% authored_guide.tex
% 2011/06/23, v3.1 gamma
%
% Adapted by Diana Gillooly and David Tranah
% from Ali Woollatt's original documentation for cambridge7A

\RequirePackage{listings}

\NeedsTeXFormat{LaTeX2e}[1996/06/01]

\documentclass{cambridge7A}
% \documentclass[spanningrule]{../cambridge7A}% option

%\todone{Don't know if this works? -- yes. \part causes problems so used \part*. \parts don't show up in ToC though}
\usepackage[linkcolor=black]{hyperref}

% \usepackage{todonotes}

\lstset{
  basicstyle=\small\ttfamily,
  columns=fullflexible,
  breaklines=true,
  frame = single,
  numbers = left,
  numberstyle = \small,
  framexleftmargin=0pt,
  language=Python
}

\newcounter{projectCounter}

%\done{Sidecap not part of this style? Keeping for the moment}
\usepackage[rightcaption,raggedright]{sidecap}% for side captions
\usepackage{framed}         % for floatingboxes
\usepackage{soul}           % for letterspacing in theorem-style headings

  \usepackage{natbib}
% \usepackage[numbers]{natbib}% option

  \usepackage[figuresright]{rotating}
  \usepackage{floatpag}
  \usepackage{mathtools}
  \rotfloatpagestyle{empty}
  
  \usepackage{booktabs}


% \usepackage{amsmath}% if you are using this package,
                      % it must be loaded before amsthm.sty
  \usepackage{amsmath}
  \usepackage{amsthm}
  \usepackage{amsfonts}
  \usepackage{graphicx}
  
  \usepackage{wrapfig}
  
\usepackage{tabularx}


\usepackage{cleveref}

\usepackage{tikz}
\usetikzlibrary{positioning,chains,fit,shapes,calc,arrows.meta}
\usetikzlibrary{decorations.pathreplacing}
\usetikzlibrary{fadings}

 \usepackage{txfonts}
 \usepackage{siunitx}
% \usepackage[scaled=0.9]{couriers}% use if you're using \tt fonts

% indexes
% uncomment the relevant set of commands

% for a single index
\usepackage{makeidx}
\makeindex

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}


% for multiple indexes using multind.sty
  \usepackage{multind}\ProvidesPackage{multind}
  \makeindex{authors}
  \makeindex{subject}

% for glossary entries
  %\usepackage[style=list]{glossary}
  \usepackage[number=none]{glossary}
\makeglossary

% theorem definitions
% see chapter 3 for details
%   \theoremstyle{plain}% default
%   \newtheorem{theorem}{Theorem}[chapter]
%   \newtheorem{lemma}[theorem]{Lemma}
%   \newtheorem{proposition}[theorem]{Proposition}
%   \newtheorem{corollary}[theorem]{Corollary}
%   \newtheorem{conjecture}[theorem]{Conjecture}

%   \newtheorem*{theorem*}{Theorem}
%   \newtheorem*{lemma*}{Lemma}
%   \newtheorem*{proposition*}{Proposition}
%   \newtheorem*{corollary*}{Corollary}
%   \newtheorem*{conjecture*}{Conjecture}

%   \theoremstyle{definition}
%   \newtheorem{definition}[theorem]{Definition}
%   \newtheorem{example}[theorem]{Example}
%   \newtheorem{prob}[theorem]{Problem}
%   \newtheorem{remark}[theorem]{Remark}
%   \newtheorem{notation}[theorem]{Notation}
%   \newtheorem{exer}[theorem]{Exercise}

%   \newtheorem*{definition*}{Definition}
%   \newtheorem*{example*}{Example}
%   \newtheorem*{prob*}{Problem}
%   \newtheorem*{remark*}{Remark}
%   \newtheorem*{notation*}{Notation}
%   \newtheorem*{exer*}{Exercise}



% \hyphenation{docu-ment baseline-skip polar}

% for this documentation, table of contents lists up to subsection level
  \setcounter{tocdepth}{2}

  \newcommand\cambridge{cambridge7A}

% remove the dot and change default for enumerated lists
  \def\makeRRlabeldot#1{\hss\llap{#1}}
  \renewcommand\theenumi{{\rm (\roman{enumi})}}
  \renewcommand\theenumii{{\rm (\alph{enumii})}}
  \renewcommand\theenumiii{{\rm (\arabic{enumiii})}}
  \renewcommand\theenumiv{{\rm (\Alph{enumiv})}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% \includeonly{06authored}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\empt}[2]{$#1^{\langle #2 \rangle}$}

\begin{document}

  \title{Personalized Machine Learning}
  \author{Julian McAuley}


%   \bookabstract{This is the guide for authors who are preparing written,
%     rather than edited, books.}
%   \bookkeywords{\LaTeX; authored books; CUP style; cambridge7A.cls.}

  \frontmatter
  \maketitle
  \tableofcontents
  %\done{Dunno if this is part of 7A. Leaving in for now}
  \listoffigures
  \listoftables
% \listofcontributors

%   \include{authored_preface}
  
  
  %\done{Dunno if this or glossary? Keep for now}
  \chapter*{Notation}

\begin{tabular}{@{}ll}
 $u \in U$ & users $u$ in user set $U$\\
 $i \in I$ & items $i$ in item set $I$\\
 $I_u$ & set of items rated by user $u$\\
 $U_i$ & set of users who have rated item $i$\\
 $|U|$ & number of users\\
 $|I|$ & number of items
\end{tabular}
%\todo{Write}

  \mainmatter
  
  
  \part*{Introduction To Personalized Machine Learning}
  %\done{Change all these names. Done but could still move them}

\chapter{Introduction}
\label{chap:intro}

Traditionally, \emph{machine learning} encompasses a broad range of problems ranging from detecting objects in an image, to finding documents relevant to a given query, to predicting the next element in a sequence, among countless others. Roughly speaking, the field of machine learning approaches these problems by mining patterns in data in order to discover some underlying `truth' about a document, image, or sequence.

Increasingly, though, 
%there is an increasing need to develop machine learning applications 
there is a need to apply machine learning in settings where the `correct' outcome is subjective, and depends on the context and characteristics of an individual user.
%
As we browse online for movies to watch, products to buy, or romantic partners to connect with, we are likely engaging with a form \emph{personalized} machine learning: that is the results are tailored to us specifically, based on the types of movies, products, or partners that \emph{we specifically} are likely to engage with.

%More complex applications: Heart rate profiles, anaesthetic volume, surgery scheduling


Much like traditional machine learning algorithms, \emph{personalized} machine learning algorithms are at their heart essentially forms of pattern discovery.
%At their heart, 
That is, predictions are made \emph{for you} by analyzing the behavior of people \emph{similar to you}. A feature such as `people who liked this also liked' (\Cref{fig:?}) is perhaps the most simple example of this type of personalized pattern discovery:\footnote{Though strictly speaking maybe not one that we'd call `machine learning.'} based on the contextual attribute of a user liking a particular item, recommendations are extracted based on users who share the same attribute. At the other end of the spectrum are complex deep learning approaches that learn `black box' representations of users in order to make predictions, though these too at their heart rely on the intuition that `similar' users (in terms of these complex representations) will have similar interaction patterns.



\section{Purpose of This Book}

In this book we seek to introduce \emph{Personalized Machine Learning} by exploring the types of approaches used to solve the above problems, and construct a narrative around the underlying methods and design principles.
%
As we explore in the next chapter, even in applications as diverse as song recommendation, or heart-rate profiling, there is a common set of techniques around which personalized machine learning systems are built.


%\todo{rearrange}


%Beyond introducing the basic techniques, 
By introducing this underlying set of principles,
%The purpose of \emph{personalized machine learning} 
the book is intended
is to teach readers how modern machine learning techniques can be improved by incorporating ideas from personalization and user modeling, and to guide readers in building machine learning systems where accurately modeling the users involved is key to success.

There is currently an abundance of models, datasets, and applications that seek to capture human dynamics or interactions. Examples pervade in diverse areas including web mining, recommender systems, fashion, dialog systems, personalized health, etc.
%\todo{could update to reflect final topics}

As such, there is an emerging set of common techniques that are used to capture the dynamics of `users' in each of these settings. This book is designed to act as a reference point to explore and explain these techniques.

As a starting point, we will begin the book (\Cref{chap:basics}) with a primer of machine learning, and especially supervised learning, that will bring readers up-to-speed on the basic techniques required later. Although this introductory material is likely familiar to many readers, we have a parcitular focus on user-oriented datasets, and show that even with `standard' machine learning techniques, there is considerable scope for building personalized systems through careful feature engineering strategies that capture relevant user characteristics.

Following this, our main introduction point to personalized machine learning will be to explore personalized recommendation (\Cref{chap:recsys}).
Recommendation technology has traditionally relied on personalization and user modeling, whether through simple similarity functions among users (`people like you also bought' etc.)~or through more modern approaches, including approaches based on neural networks.

More recently, the need to account for personalization and to model users has encroached on a variety of other areas of machine learning. Exploring personalization and user modeling in these new areas---and more broadly giving readers the tools they need to design personalized approaches to them---is the main goal of this book.
%\todo{badly worded}

Examples include:
\begin{description}
\item[Sequence Modeling and Time-Series Analysis]
\item[Natural Language Processing] Modern NLP methods focus on interactive systems for language synthesis, dialog, question answering, etc. Within these applications there is a need to understand people's personalized writing styles, subjectivity, personal context (etc.). Understanding the variation
among users is key to success in these applications.
\item[Computer Vision] A few `traditional' computer vision problems depend on models of individual users (e.g.~face recognition), though we are more interested in modern applications, such as personalized models of visual preferences, applications in fashion, and personalized GANs.
%\todo{finish, change paragraph above, reorder to match chapters}
%\todo{All of these should be longer}
%\item[Online Advertising]
\item[Fairness and Ethics] Finally, there is growing recognition of the risks of blindly applying personalization and recommendation technologies without regard for the consequences of doing so. Examples include filter bubbles, `concentration' effects (e.g.~due to a lack of diversity of recommended content), or systematic biases against users from underrepresented groups. In response, there is a growing body of work that tries to understand and correct these types of bias
\end{description}


% Sequence Modeling Personalized models of sequence data appear when mod-
% eling (e.g.) activity traces, heart rate data, sleep patterns etc.

% Application domains of interest that will serve as focus areas for the book
% include:

% e-Commerce which includes many of the ‘traditional’ problems in recommen-
% dation, but also newer areas such as personalized fashion.

% Healthcare When modeling data such as heart rate sequences, clinical notes,
% (etc.) it is critical to capture the variation among individual users in order
% to make meaningful predictions.
% Web mining Models of behavior on the web, whether to predict future actions,

% model user attrition, model bias and extremism, (etc.) rely on disentan-
% gling global (or ‘background’) effects from the characteristics and varia-
% tions among individuals. This topic is closely connected to the emerging
% 
% area of computational social science.

% \section{Audience}
% The book is primarily intended for readers who already have some
% familiarity with machine learning, though basics will be revisited.
% A typical reader might be a student, researcher, or practitioner with some exposure to ML fundamentals, but who is looking to explore how their applications of interest can be improved by modeling the users involved.

% We 
% %\todo{We or I?}
% hope that any readers working in the areas described above would benefit from the
% presentation and organization of this book. For example, a practitioner working
% with clinical text, and wishing to model their data with machine learning, might
% find their needs poorly served by existing texts. With the increasing need to
% develop personalized machine learning models, it is useful and timely to have a
% textbook with this particular focus.

\section{For Learners}

Although this book is primarily intended as a guide to the specific topics of personalization, recommendation, and user modeling (etc.), it should also serve as a relatively gentle introduction to the topic of machine learning in general. Indeed, 
%\todo{Consider whether to use "I"}
courses I've taught on recommender systems have routinely served as students' first exposure to the broad topic of machine learning. Such material is ideal as a starting point for learners seeking a more `application oriented' view of machine learning compared to what is typically covered in introductory machine learning texts.

\subsection{Expected Knowledge}
\label{sec:expectedKnowledge}

While suitable as a first book on machine learning, the material in this book is intended for readers with some background in computer science, especially some familiarity with statistics, probability, linear algebra, and some experience working with structured datasets (e.g.~\emph{json}, \emph{csv}).
%\todo{style}

%\todo{connecting sentence}
\begin{itemize}
    \item Programming in \emph{Python}, which is used for all code examples.
    \item Basic string and data processing, file I/O etc.
    \item A basic knowledge of linear algebra, probability, etc.
\end{itemize}

%Links to resources page
%moved
%To help readers with exercises and projects, and to collect resources such as datasets and additional reading materials, we have prepared an online supplement:

% \begin{center}
% \url{http://cseweb.ucsd.edu/~jmcauley/pml/index.html}
% \end{center}



For readers less familiar with the above, we give links on our online resources page (\Cref{sec:onlineResources}).
%\todo{finish}

\subsection{What is \emph{Not} Covered}

\begin{description}
    \item[Regressors and classifiers] 
    %\todo{Consistent use of title case throughout book}
    When introducing basic machine learning concepts in \Cref{chap:basics}, we limit ourselves to linear regression and linear classification (logistic regression), since these serve as building blocks for later methods we develop. Consequently, we ignore dozens of alternative regression and classification methods that are often the core of standard machine learning texts, though we discuss the merits of these alternatives in \Cref{sec:?}
    \item[Dimensionality reduction ]
    We avoid a detailed treatment of dimensionality reduction and clustering algorithms, both staples of many introductory texts on machine learning. We note however that many of the techniques we explore when learning user representations are 
    %essentially 
    at their heart forms of dimensionality reduction. We 
    %simply 
    choose to avoid a linear algebra-heavy presentation of `traditional' dimensionality reduction techniques (principal component analysis, singular value decomposition, etc.),~which in terms of actual implementation have little in common with the methods we develop.
    %\todo{not great}
    \item[Deep learning] Although we cover various deep learning techniques (e.g.~multilayer perceprtron-based recommendation in \Cref{sec:?}, sequence models based on 
    recurrent neural networks
    %RNNs 
    in \Cref{chap:temporalSequential}, and models of visual preference based on 
    convolutional neural networks
    %CNNs
    in \Cref{chap:visual}), 
    by doing so we are merely scratching the surface of deep learning-based personalization.
    However, we note that deep learning approaches are merely one specific solution to the same problems that we visit using more traditional techniques.
    %\done{funny sentence above}
    %\todo{More?}
    We survey deep learning-based personalization techniques in \Cref{sec:?}.
    \item[Reinforcement and online learning]
    We largely limit ourselves to traditional supervised learning problems, i.e.,~uncovering patterns and making predictions from historical collections of training data. In practice when deploying predictive models, data is obtained in a streaming setting and updates must be made in real time. This type of training regime is known as \emph{online learning}, which we briefly cover in \Cref{sec:onlineUpdates}. We also avoid discussion of reinforcement learning algorithms,
    %which are used in settings including personalized
    though touch upon their use for online advertising in \Cref{sec:?}.
    \item[User interfaces and human computer interaction]
    %Finally, we note that our study of personalization is 
    By design
    largely confined to machine learning approaches. That is, we are generally concerned with building predictive systems that can estimate---as accurately as possible---how a particular user will respond to a given stimulus. By doing so, we can estimate preferences, predict future activities, retrieve relevant items, etc. This is complementary to a large body of work that explores personalization from the perspective of human computer interaction, where the primary concern is maximizing the quality of the user experience (ease of finding information, satisfaction, long-term engagement, etc.). \item[Deployment]
    Likewise we largely avoid the `systems building' side of personalized machine learning, such as concerns around deploying machine learning models on distributed servers (etc.),~though we discuss high-level libraries and implementation best-practices throughout the book.
\end{description}

%\todo{paragraphs not connected}

%Of course, there are 
Of course, we are mindful of the
dangers associated with `black-box' approaches to machine learning,
and want to avoid the pitfalls of
blindly optimizing model accuracy, such filter bubbles, unwanted biases, or simply degraded user experience. In \Cref{chap:ethics} we discuss these issues, as well as potential approaches to address them. However our discussion is still limited to machine learning solutions, i.e.,~we investigate \emph{algorithmic} approaches to correct for biases, increase recommendation diversity, etc. 
We note that algorithmic solutions are only part of the picture, and that while having better algorithms is critical, it is also critical that those algorithms are appropriately \emph{used}. Such questions fall under the umbrella of human computer interaction, and user interface design. Although these topics are outside the scope of this book, we discuss them briefly at the end of
%\todo{Either make this exist or delete}
\Cref{chap:ethics}.

\section{For Instructors}

%\emph{Personalized Machine Learning}
%\done{looks kinda goofy to use book title in sentence}
This book
is inspired by my own experience teaching classes on recommender systems and web mining at UC San Diego. Courses on these topics have proved extremely popular, and are often chosen as learners' first machine learning courses.

One reason this topic acts as a good first contact with the machine learning curriculum is that it has a somewhat lower bar for entry than many machine learning courses, including (for example) courses on deep learning, or even many introductory machine learning classes. Partly this is due to the material being less dependent on deep and complex theory, and partly it is due to the ability to quickly build working solutions that are fairly representative of the state-of-the-art, rather than mere proof of concepts. As such, a focus of this book is to quickly build working solutions, and covering a wide breadth of approaches, rather than diving too deep into the theory behind any one approach. I believe this will be useful in helping learners
%\done{always use word "learners" or "students". Looks consistent, student only used when needed}
to understand the practical considerations behind building predictive systems based on user data, and will be complementary to the more theoretical treatment given in most introductory texts.

Another feature that has made this material popular among learners is the ability to work quickly with large, real-world datasets. The ability to work with collections of user data from \emph{Amazon}, \emph{Google}, \emph{Steam} (etc.),~on applications that are representative of real use-cases, has proved immensely valuable for students building their project portfolios or preparing for interviews. As such, each chapter is paired with project suggestions, each of which would be suitable as a major class project. These projects aim to synthesize the material from each chapter, with a particular focus on `system building' considerations, design choices, and thorough model evaluation.

\subsection{Course Plan and Overview}

%\todo{redundant?}
The content in this text is aimed at developing a quarter- or semester-long course, for students with some background in linear algebra, probability, and data processing. Chapters \Cref{chap:basics} and \Cref{chap:recsys} cover core material upon which the remainder of the book builds.
Chapters
\Cref{chap:temporalSequential} to \Cref{chap:visual}
%\todo{check}
are somewhat more orthogonal, such that components can be selected and combined as time or student background allows. A final chapter on 
bias, ethics, and fairness
%\todo{check above}
(\Cref{chap:ethics}) provides an opportunity to revisit earlier material through a new lens.

%\todo{Say something about projects and homeworks here}
%Projects, homeworks
Each chapter is paired with homework and projects

%\begin{description}
  \paragraph{Machine Learning Primer} (\Cref{chap:basics}) 2-3 weeks.
  %a selection of datasets that include user interaction data, in order to visit (or revisit)
  Introduces the foundational concepts of machine learning, feature design, and evaluation, via a selection af datasets that capture user interactions. Exercises (\Cref{sec:basicsExercises}) range from simple data manipulation to building a working machine learning pipeline (training, validation, etc.). Exercises in \Cref{chap:basics} are mainly concerned with feature design, including a project (Project~\ref{proj:1})
  %\todo{Could make this work with Cref if I can be bothered?}
  that involves experimenting with activity data involving temporal and geographical dynamics.
  %\done{Is this accurate?}
  
  \paragraph{Recommender Systems} (\Cref{chap:recsys}) 2-3 weeks. In \Cref{chap:recsys} we introduce the core set of techniques used for recommendation. Traditional heuristics are presented along with machine learning approaches. Recommender systems are used to develop the concept of a \emph{user manifold} which is used throughout the following chapters to capture variation among users in various other settings.
  %\todo{not great explanation. "various other settings" sounds generic}
  Exercises 
  are mainly concerned with the basics of building practical recommendation approaches, 
  and a project (Project~\ref{proj:2}) is concerned with building an end-to-end recommendation pipeline for a book recommendation scenario.
  %\done{Is that what's actually in the project}
  
  \paragraph{Content and Structure in Recommender Systems} (\Cref{chap:contentStructure}) 1 week. Explores how to incorporate features (i.e.,~side information) into personalization (mostly recommendation) approaches, and explores personalization in settings with additional structure, such as socially-aware recommendation, news recommendation, etc.
  %\todo{confirm that's what's actually covered}
  Some of these content-aware approaches (such as factorization machines) are revisited later the book when 
  %incorporating othe
  developing more complex models based on (e.g.)~temporal or sequential dynamics. A project (Project~\ref{proj:3}) consists of developing recommender systems for use in cold-start settings (i.e.,~capable of adapting to new users or new items).
  
  \paragraph{Temporal and Sequential Models} (\Cref{chap:temporalSequential}) 1-2 weeks. Starts by revising some of the basic approaches to temporal and sequential modeling, such as autoregression and Markov chains; later explores how to personalize such models, 
  %ranging from simple
  and explores complex approaches based on recurrent networks.
  %\todo{confirm that that's true}
  A project (Project~\ref{proj:?}) compares various approaches to temporal recommendation.
  %\todo{tense of these sentences not very consistent}
  
  \paragraph{Personalized Models of Text} (\Cref{chap:text}) 1 week. After revising some of the basic predictive models of text (such as bag-of-words representations), we explore how text can be used to understand the dimensions of preferences.
  We revisit sequential modeling by exploring techniques that borrow from natural language to model interaction sequences. We also visit methods for text \emph{generation}, which can be personalized in settings ranging from conversation to justification of machine predictions. A project (Project~\ref{proj:?}) consists of building personalized systems for document retrieval.
  
%   \paragraph{Online Advertising}
  %\done{Does this chapter exist? No}
  
  \paragraph{Personalized Models of Visual Data} (\Cref{chap:visual}) 1 week. Explores applications involving visual data, ranging from personalized image search, to applications in fashion and design. Exercises and a project (Project~\ref{proj:visualProj})
  %\todo{don't write exercises and a project and then only reference the project}
  consist of building visually-aware recommendation systems for applications in fashion.
  
  \paragraph{The Ethics of Personalized Machine Learning} (\Cref{chap:ethics}) 1-2 weeks. The final chapter
  %\todo{Not the final chapter, possibly rewrite}
  looks at the ethics and the various pitfalls of developing personalized machine learning systems. Examples include filter bubbles, extremification, and issues of bias and fairness. The chapter has a significant focus on applied case-studies, and allows us to revisit several of the topics from previous chapters through a new lens. Exercises and a project (\Cref{proj:fairBeer}) consist of improving recommendation approaches in terms of gender parity and other fairness objectives.
%\end{description}
%\todo{Make sure items match contents}
%\todo{Repetitive, isn't there another section saying basically the same thing?}
%\todo{Could just expand to several paragraphs and not make it a "course outline" type thing}

% \paragraph{Related Topics and Further Reading}
% %\done{DELETE Chapter probably not called that}
% (\Cref{chap:further}) 1 week. The last chapter points to further directions that are closely related to personalization, but do not quite fit elsewhere. This chapter includes discussion of topics that are closely related to personalized machine learning, such as ad recommendation, but which rely on techniques quite different from those covered in the book (such reinforcement learning), as well as topics for further reading that go beyond machine learning approaches, such as user interface design and deployment considerations.
% %\done{DELETE Rewrite once the topics are in the chapter}
% The material in this chapter is perhaps ideal as `bonus content,' from which a few interesting topics may be drawn, allowing some flexibility into lecture scheduling.


\section{Online Resources}
\label{sec:onlineResources}

%To supplement the material covered in this text, an online supplement is available to augment the material covered here with working code and examples.
To help readers with exercises, projects, and to collect resources such as datasets and additional reading materials, 
%we have prepared an online supplement:
an online supplement is available to augment the material covered here with working code and examples:
%\todo{could say more}
%
%These resources are collected on 
\begin{center}
\url{https://cseweb.ucsd.edu/~jmcauley/pml/}
%\todo{make sure only https and that links work}
\end{center}

The online supplement includes:
\begin{itemize}
    \item Complete code examples covering the material in each chapter. These cover complete worked examples from which the code samples presented in each chapter are drawn. Additional code samples are included that reproduce various figures and examples presented throughout the book.
    \item Links to all datasets used in the book (as well as various other personalization datasets).
    \item Links to additional reading, mostly focused on introductory material useful to learners less familiar with some of the background material described in \Cref{sec:expectedKnowledge}.
\end{itemize}
%\todo{make sure above exists and covers all relevant items}

\section{About the Author}

Julian McAuley has been an Assistant Professor at UC San Diego since 2014, and an Associate Professor since 2019. Previously he was a postdoctoral scholar at Stanford University, after completing my undergraduate and graduate training at the University of New South Wales and the Australian National University.
%\todo{boring!}

\begin{wrapfigure}{r}{0.4\linewidth}
\includegraphics[width=\linewidth]{fig/portrait_bw.jpg}
\caption{The Author}
\end{wrapfigure}

\emph{Personalized Machine Learning} is the main research theme of my lab at UCSD. Our research has pioneered the use of images and text in recommendation settings (e.g.~\cite{mcauley2015image,mcauley2013hidden}), with applications including fashion design, personalized question answering systems, and interactive dialog systems. Our lab has also studied personalization outside of typical recommendation settings, such as developing personalized models of heart-rate profiles \cite{ni2019modeling}, and systems for generating personalized recipes \cite{majumder2019generating}.

\chapter{Personalization in Everyday Life}
%\todo{Chapter has little value, could fold into previous chapter}

Other than introducing the techniques that underlie personalized machine learning systems, one of our goals in this book is to explore the wide range of practical applications where personalization is applied, to explore the history of the topic, and eventually to explore the broad consequences and ethical concerns associated with personalized machine learning.

\section{The Importance of Personalization}

%\subsection{E-Commerce and Advertising}
\subsection{Recommendation}

Many of the examples we cover in this book will relate to \emph{recommender systems}, and more broadly modeling user interactions on the web.
%on e-Commerce platforms, using data
%largely 
Part of the reason for this focus is opportunistic: user interaction data is widely available, allowing us to build models on top of real data throughout the book.
%
%They

Pedagogically, recommender systems are also appealing as an introduction to personalized machine learning as they allow us to quickly implement working systems that are close to the state-of-the-art. As we'll see, even widely-deployed systems turn out to be surprisingly simple, relying on simple heuristics and building on standard data structures (\Cref{sec:amazonCaseStudy}).
%\todo{finish}

Ultimately though, our main reason for studying recommender systems is because they are a fundamental tool for modeling \emph{interactions between users and items}. As such, the basic techniques developed when building recommender systems can be applied in a variety of situations where we want to predict how a user will respond to some stimulus. We explore a variety of related settings below.

\subsection{Social and News-Media}
%\subsection{User Productivity}

%\section{Personalization Beyond E-commerce}
\section{Personalization Beyond Recommendation}

Fundamentally, the value of \emph{personalized machine learning} is to model datasets where \emph{variation among individuals} captures a large fraction of the variability in a dataset.
%\todo{already said something like this?}
%\todo{finish, above sentence doesn't really fit either}
%
A high degree of variability among individuals manifests itself in a wide variety of settings. In recommender systems, users might vary due to subjective preferences, budgets, or demographic factors; in settings like personalized health, users may vary in terms of their physical characteristics, medical histories, or risk factors; or in settings involving natural language (or dialog) users may vary in terms of their writing styles, personalities, or their specific context.

Below we describe a few such examples, partly to highlight the wide range of settings where personalization is critical, but also to demonstrate the common set of ideas involved in modeling them.
%\done{not the best written. Is fine}

\subsection{Personalized Health}
%\done{Could drop to last chapter and flesh out a bit, or maybe not. No way}
%\todo{finish}

\emph{Personalized health}

Note that most of these tasks are ones for which \emph{non}-personalized models would not be effective
%\todo{put somewhere}

Estimating what symptoms a patient will exhibit on their next hospital visit is a canonical task in personalized health, with applications in (e.g.)~preventative treatment. This task closely resembles the settings we explore when developing recommender systems, given the goal to estimate patients' interactions with certain stimuli (symptoms) over time \cite{yang2014finding}. As such, techniques for such tasks borrow ideas from recommender systems, especially temporal and sequential recommendation, as we develop in \Cref{chap:temporalSequential}.
%\todo{maybe cite this again when I talk about beer paper in temporal chapter}

Beyond estimating patient symptoms, personalized machine learning techniques can be adapted to related tasks such as estimating the duration of surgical procedures (in order to more efficiently utilize hospital resources). Accurate prediction for such a task depends on both patient and surgeon characteristics. Successful models for this task have been built using traditional regression techniques, along with carefully-engineered patient and surgeon features
\cite{ng2017predicting}, resembling the types of personalization we explore in `standard' regression and classification settings in \Cref{chap:basics}.

Beyond regression, classification, and ranking tasks, a variety of problems in personalized health depend on more complex predictions, e.g.~forecasting sequential data. For example, techniques to model the progression of heart-rate sequences in response to physical stimuli
%Heart-rate modeling
\cite{ni2019modeling},
or to estimate the distribution kinetics of drugs (such as anesthetics)
%Drug dose prediction
\cite{ingrande2020performance},
requires complex sequential models. Modern approaches to these types of sequential modeling tasks are often based on recurrent neural networks, which are capable of learning complex semantics from sequential data; we explore these types of personalized sequential models in \Cref{sec:?}.

%Clinical note generation
Many problems in personalized health also depend upon natural language data, for example modeling the characteristics of clinical notes, which again exhibit substantial variation as a function of the patient or physician. More complex task may even consist of generating notes automatically based on (e.g.)~a radiology image
\cite{ni2020learning}. Such applications build on techniques for personalized natural language processing and generation, as we develop in \Cref{chap:text}.

%\todo{mention how above techniques represent the different "types" of personalized learning}

\subsection{Computational Social Science}

Often the goal of modeling user data is not merely to \emph{predict} future events or interactions, but to \emph{understand} the underlying dynamics at play. As such

\begin{description}
 \item[Price sensitivity]
 \item[Spotify maybe]
\end{description}

\subsection{Language Generation, Personalized Dialog, and Interactive Agents}

Finally,
%\todo{is it finally?}
given the new modalities 
via which people interact with predictive systems, there are new demands for personalization.

%Beyond dialog, 
For example, 
personalization is critical in a broad range of settings involving natural language. User-generated language data exhibits substantial variability due to differences in writing style, subjectivity, etc. When dealing with such data, non-personalized models may struggle with this nuance.

For example, automated systems for dialog, whether in task-oriented settings or for open-domain `chit-chat' can benefit from personalization, in order to generate responses that are more personalized or empathetic to tone or context of individual users
%
\cite{majumder2020like}.

%\cite{mao2020speech}
%probably not relevant





\section{The Risks and Ethics of Personalization}
%\todo{rename}

Along with the increasing ubiquity of personalized machine learning systems, there is a growing awareness of the risks associated with personalization. Some of these issues have reached mainstream awareness, such as the idea that personalized recommendations can trap users in `filter bubbles,' while other issues are considerably more subtle.

%\todo{stuff}

For instance, considering the specific case of recommender systems, a na\"ively-implemented model can introduce issues including:
\begin{description}
    \item[Filter bubbles] Roughly speaking, recommendation algorithms rely on identifying specific item characteristics that are preferred by each user, and recommending items that most closely represent those characteristics. Without care, even a user with broad interests may be recommended only a narrow set of items that closely mimic their prior interactions.
    \item[Extremification] Likewise, a system that identifies features that a user is interested in may identify items that are most representative of those features, e.g.~a user who likes action movies may be recommended movies with \emph{a lot} of action; in contexts such as social media and news recommendation this can lead to users being exposed to increasingly extreme content (the relationship between this and the previous issue is explained in \Cref{chap:ethics}).
    \item[Concentration] Similar to the previous phenomenon, a user who has diverse interests may receive recommendations that only follow their most predominant interest (\Cref{sec:filterBubble}). In aggregate, this may lead to to a small set of items being over-represented among users' recommendations. 
    \item[Bias] Given that recommenders (and many other personalized models) ultimately work by identifying common patterns of user behavior, users in the `long-tail' whose preferences don't follow the predominant trends may receive sub-par recommendations.
\end{description}

Along with a rising awareness of these issues has come a set of techniques designed to mitigate them. These techniques borrow ideas from the broader field of fair and unbiased machine learning, whereby learning algorithms are adapted so as not to propagate (or not to exacerbate) biases in training data, though the fairness goals are often quite different. 
Diversification techniques can be used to ensure that predictions or recommendations balance relevance with novelty, diversity, or serendipity;
%One set of techniques seeks 
related techniques seek
%\done{Clunky wording}
to better `calibrate' personalized machine learning systems by ensuring that predicted outputs are balanced in terms of categories, features, or the distribution over recommended items (\Cref{sec:diversification}). 
%\todo{correct section?}
Such techniques can mitigate filter bubbles by ensuring that model outputs aren't highly concentrated around a few items, and more qualitatively can increase the overall novelty or `interestingness' of model outputs. Other techniques follow more directly from fair and unbiased machine learning, ensuring that the performance of personalized models is not degraded for users belonging to underrepresented groups, or who have niche preferences (\Cref{sec:fairness}).

\section{Techniques for Personalization}

As mentioned in \Cref{chap:intro}, one of the goals of this book is to establish a common narrative around the tools and techniques used to design personalied machine learning systems. Although we've shown that such systems are applied in domains as diverse as e-Commerce to personalized health, we find that the techniques used to implement these models follow a few common paradigms.

\subsection{User Representations as Manifolds}

One of the main ideas we'll revisit throughout this book---and which allows us to adapt ideas from recommender systems to other types of machine learning---is that of a \emph{user manifold}. That is, most of the personalized methods we'll explore will involve \emph{representations} of users that describe the common patterns of variation in their activities and interactions.

In the case of recommender systems, this `user manifold' will be a vector that describes the principal dimensions that explain variance among user preferences (\Cref{fig:recsysIdea}). For example, we might discover that the principal 
%\done{Check principle / principal throughout}  
dimensions that explain variance in preferences in a movie recommendation setting
center around certain genres, actors, or special effects. 
%In another setting, 
In a text generation setting (\Cref{sec:?})
%\done{not great}
user manifolds might describe patterns of variation in users' writing styles; when estimating users' heart-rate profiles (\Cref{sec:?}), user manifolds might describe users' fitness levels or other patterns of variation in heart-rate dynamics;
or in a personalized fashion setting (\Cref{sec:?}), user manifolds might describe users' preferences toward specific visual characteristics.
%\todo{says same thing as below}

%\done{Make sure no \cref (lowercase c) exists}

\begin{figure}
\centering
\includegraphics[scale=.75]{fig/recsysIdea.pdf}
\caption{The basic idea behind recommender systems, and various other types of personalized machine learning, is to represent users by \emph{low-dimensional manifolds} that describe the patterns of variance among their interactions. \label{fig:recsysIdea}}
\end{figure}

Throughout the book, we'll revisit the idea of user manifolds, as a general-purpose means of capturing common patterns of variation among users. Some examples include:
\begin{itemize}
    \item In \Cref{chap:recsys}, we'll use low-dimensional user representations to describe the dimensions of preferences and activities, which can be used to recommend items that users are likely to interact with.
    \item In \Cref{chap:text}, user representations can describe the topics users tend to discuss (e.g.~when writing reviews), or individual characteristics of their writing styles.
    \item In \Cref{chap:visual}, user representations will describe the visual dimensions that users are interested in, allowing us to rank, recommend, or generate images in a personalized way.
    \item Throughout various case studies, user representations will capture characteristics ranging from dietary preferences (\Cref{sec:recipeCaseStudy}), fitness profiles (\Cref{sec:caseStudyHR}), social trust (\Cref{sec:?}), or fashion choices (\Cref{sec:fashionCaseStudies}).
    %\todo{fashionCaseStudies isn't the perfect ref}
\end{itemize}

\subsection{Contextual Personalization and Model-Based Personalization}
\label{sec:contextVsPerson}

Although this book will predominantly cover methods that explicitly model user terms (as above), we will also cover a variety of models that deliberately avoid doing so.

Starting with trivial models such as `people who bought X also bought Y,'
%\todo{what do I call this elsewhwere?}
many classical approaches for (e.g.)~recommendation \emph{leverage user data}, but do not include explicit parameters (i.e.,~a `model') associated with a user. However, such models are still \emph{personalized}, in the sense that different predictions will be made for each individual based on how they interact with the system. Simple machine learning techniques, such as those we develop in \Cref{chap:basics}, where users are represented by a few carefully-engineered features, also follow this paradigm. 

We'll distinguish between these two classes of approach using the terms \emph{model-based} and \emph{contextual} personalization. \emph{Model-based} approaches learn an explicit set of parameters associated with each user, such as the `user manifolds' described above; these models are typically intended to capture 
the predominant patterns of variation among users in a system, usually in terms of a low-dimensional vector.
In contrast, \emph{contextual} (also sometimes called `memory-based,' as in \Cref{chap:recsys}) approaches extract features from users' histories of recent interactions.

There are several settings in which contextual personalization may be preferable to explicitly modeling a user. 
%For example, 
When developing simple recommender systems in \Cref{chap:recsys}, and even more trivial personalized models in \Cref{chap:basics}, we see that personalization can often be achieved with simple heuristics, or hand-crafted features or similarity measures. Such approaches may be desirable for a number of reasons: simple models may be more interpretable (and therefore preferable to expose to a user compared to `black-box' predictions); or, we may lack adequate training data to learn complex representations from scratch.
%\todo{Doesn't finish as punchily as it could}


\part*{Fundamentals of Personalized Machine Learning}


\chapter{Machine Learning Primer}
\label{chap:basics}

In this chapter, we'll cover the fundamental principles of machine learning (and in particular \emph{supervised} learning), that will serve as a foundation for the remaining material in this book.

Although we'll only briefly touch upon \emph{personalization}
in this chapter, 
our examples will focus on the same types of user-oriented data that we'll visit in later chapters. In particular, we'll focus on datasets covering topics such as recommendation, sentiment, and predictive tasks involving (e.g.)~demographic characteristics.

As such, the view we'll take on `personalization' in this chapter will consist of \emph{extracting features about individuals} in order to make predictions using traditional machine learning frameworks. Later, we'll draw a distinction between this type of method---where we extract \emph{features} about users---and methods where we explicitly \emph{model} each user. This will drive our discussion of \emph{contextual} versus \emph{model-based} personalization, though we'll discuss this distinction more precisely in \Cref{sec:modelbased}. However, as we'll see in this chapter (and in various examples throughout the book), even traditional machine learning techniques, paired with appropriate feature extraction strategies, can lead to surprisingly effective models for personalized prediction.
%\todo{not the tightest writing}
%\todo{When do I actually clarify the distinction between types of personalization? }

%\todo{finish. Needs more intro}

\paragraph{Supervised Learning}
%\done{Is paragraph the correct level?}

Most of the personalization techniques we'll explore throughout this book are forms of \emph{supervised learning}.
%Supervised learning techniques assume that there is a 
Supervised learning techniques assume that our prediction tasks (or our datasets) can be separated into two components:
\begin{description}
\item[labels] (denoted $y$) that we would like to predict, and
\item[features] (denoted $x$) which we believe will help us to predict those labels
\end{description}
%\todo{vector or label? x or X?}
For example, given a sentiment analysis\index{Sentiment Analysis} task (Chapter \ref{chap:text}), our data might be (the text of) reviews from \emph{Amazon}\index{Amazon} or \emph{Yelp}\index{Yelp}, and our labels would be the ratings associated with those reviews.

Given this distinction between features and labels in a dataset, the goal of a \emph{supervised learning} algorithm is to infer the underlying function
%\begin{equation}
    $f(x) \rightarrow y$
%\end{equation}
that explains the relationship between the features and the labels. Usually, this function will be \emph{parameterized} by model parameters $\theta$, i.e.,
\begin{equation}
    f_\theta(x) \rightarrow y.
    \label{eq:firstEquationTheta}
\end{equation}
For example, in this chapter $\theta$ might describe which features are positively or negatively correlated (or uncorrelated) with the labels, or later $\theta$ might capture the preferences of a particular user in a recommender system (\Cref{chap:recsys}). \Cref{box:supervised} explains how this type of approach relates to other types of learning.
%\todo{above sentence is weird}

\begin{figure}
\fbox{
\begin{minipage}{\linewidth}
% \hbox{
\emph{Supervised Learning} approaches are those that seek to directly learn the relationship between the observed data $X$ and the labels $y$. 
%Examples include regression and classification, as covered in this chapter.
Most of the models in this book are forms of supervised learning, starting with regression and classification in this chapter, and continuing into later chapters as we build models to predict user activities.
In contrast, \emph{unsupervised learning} approaches seek to find patterns in the data $X$, but are not specifically concerned with predicting any label; examples include techniques for clustering and dimensionality reduction. 
%\done{Covered in some chapter? No}
Finally, \emph{semi-supervised learning} approaches are somewhere in between, usually leveraging large datasets of \emph{unlabeled} data to improve the performance of supervised models with a small number of labels.
% }
\end{minipage}
}
%\todo{Poor examples}
\caption{Supervised, Semi-supervised, and Unsupervised Learning}
\label{box:supervised}
\end{figure}

Throughout this chapter, we will assume that we are given \emph{labels} in the form of a vector $y$, and features in the form of a matrix $X$, so that each $y_i$ is the label associated with the $i^{\text{th}}$ datapoint, and $x_i$ is a vector of features associated with that datapoint.

%\section{Regression}
%\done{Section structure? Regression could be its own chapter. Probably not}

\section{Linear Regression}
\label{sec:linearReg}

Perhaps the simplest association we could assume between our features $X$ and our labels $y$ would be a \emph{linear}\index{Linear model} relationship, i.e.,~that the relationship between $X$ and $y$ was defined as
\begin{equation}
  y = X\theta,
  \label{eq:linearTheta}
\end{equation}
or in our notation from \Cref{eq:firstEquationTheta}
\begin{equation}
    f_\theta(X) = X\theta.
\end{equation}
In the above, $X$ is a matrix describing features extracted from the data, $y$ is our vector of labels, and $\theta$ is a vector of unknowns which describe which features are \emph{relevant} to predicting the labels.
%\todo{almost repeated sentence. Could drop this one or the one above}

Ignoring strict notation for now, a trivial example might consist of predicting a review's rating as a function of its length. To do so, let's consider a small dataset of 100 $(\text{length}, \text{rating})$ pairs from \emph{Goodreads} Fantasy novels.
%\todo{Reference the dataset / exercise properly}
\Cref{fig:ratingLength} plots the relationship between review length (in characters) and the rating.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLength.pdf}
\caption{Ratings compared to review length (in characters), based on 100 reviews of fantasy novels from \emph{Goodreads} \label{fig:ratingLength}}
\end{SCfigure}

From \Cref{fig:ratingLength}, there appears to be an association between ratings and review length, i.e.,~more positive reviews tend to be longer. A very simple model might attempt to describe that relationship with a line, i.e.,
\begin{equation}
  \text{rating} \simeq \theta_0 + \theta_1 \times (\text{review\ length}).
  \label{eq:sysOfEq0}
\end{equation}
Note that the above is just the standard equation for a line ($y = mx + b$), where $\theta_1$ is a slope and $\theta_0$ is an intercept.

If we can identify a line that approximately describes this relationship, we can use it to estimate a rating from a given review, even though we may never have seen a review of some specific length before. In this sense, the line is a simple \emph{model} of the data, as it allows us to predict labels from previously unseen features. To do so we formalize the problem of finding a line of best fit.

Specifically, we are interested in identifying the values of $\theta_0$ and $\theta_1$ that
%correspond to a \emph{line of best fit}, i.e.,~so that the line 
most closely matches the trend in \Cref{fig:ratingLength}.
%
To solve for $\theta = [\theta_0, \theta_1]$, we can write out the problem as a system of equations in matrix form:
\begin{equation}
  y = X \cdot \theta,
  \label{eq:sysOfEq1}
\end{equation}
where $y$ is our vector of observed ratings and $X$ is our matrix of observed features (in this case the reviews lengths). For the first few samples of our Goodreads data we have:
\begin{equation}
  \underbrace{\left[ 
  \begin{tabular}{c} 5 \\ 5 \\ 5 \\ 4 \\ 3 \\ 5 \\ \vdots \end{tabular}
  \right]}_{y}
  \simeq
  \underbrace{\left[
  \begin{tabular}{cc} 1 & 2086 \\ 1 & 1521 \\ 1 & 1519 \\ 1 & 1791 \\ 1 & 1762 \\ 1 & 470 \\ \multicolumn{2}{c}{\vdots} \end{tabular}
  \right]}_{X} \cdot \underbrace{\left[ \begin{tabular}{c} $\theta_0$ \\ $\theta_1$ \end{tabular} \right]}_{\theta}.
  \label{eq:sysOfEq2}
\end{equation}
It is useful to compare \Cref{eq:sysOfEq0,eq:sysOfEq2} to understand how the matrix expression above expands to include the slope ($\theta_1 \times (\text{review\ length})$) and intercept ($\theta_0$) terms.

\begin{figure}
\caption{Why is there a column of `1's in the feature matrix?}
The first column of the feature matrix $X$ in \Cref{eq:sysOfEq2}, and in most feature matrices throughout this chapter, is a column of ones. To explain why we always have this feature, it is useful to expand the inner product $[1, \text{length}] \cdot [\theta_0, \theta_1]$ (e.g.~as in \Cref{eq:sysOfEq2}) to confirm that it expands to the equation for a line $\theta_0 + \theta_1 \times \text{length}$. Without the constant term in our feature matrix, we would be implicitly assuming that the fitted line passes through $(0,0)$.
%\done{Only moderately useful explanation}
\label{box:feature1}
\end{figure}

We would like to solve \Cref{eq:sysOfEq2} for $\theta$. Na\"ively, we might attempt to multiply both sides of the equation $y = \theta \cdot X$ by $X^{-1}$; however the inverse is not well-defined, since $X$ is not a square matrix.

To obtain a square matrix, we (left) multiply both sides by $X^T$:
\begin{equation}
    X^T y \simeq X^TX \theta,
\end{equation}
resulting in a square (in this case $2 \times 2$) matrix $X^TX$. We can now multiply both sides by the inverse of this matrix:
\begin{equation}
    (X^TX)^{-1}X^T y \simeq (X^TX)^{-1}(X^TX)\theta, \quad \text{or\ simply} \quad \theta = (X^TX)^{-1}X^T y.
    \label{eq:pseudoInv}
\end{equation}
The quantity $(X^TX)^{-1}X^T$ is known as the \emph{pseudoinverse} of $X$.

Computing $\theta = (X^TX)^{-1}X^Ty$ for our 100 ratings from Goodreads yields
%\todo{make sure numbers match those in figure}
\begin{equation}
    \theta = \left[ 
    \begin{tabular}{c} 
    $3.984$ \\ 
    $1.194 \times 10^{-4}$
    \end{tabular}
    \right],
    \label{eq:thetaTwo}
\end{equation}
corresponding to the line
\begin{equation}
    \text{rating} = 3.984 + 1.194 \times 10^{-4} (\text{review\ length}).
    \label{eq:reviewLength1}
\end{equation}
This line reflects a positive (albeit slight) trend between review length and ratings: for every additional character in a review, our estimate of the rating increases very slightly (by $1.194 \times 10^{-4}$ points). This line of best fit is depicted in \Cref{fig:ratingLengthBestFit}. We will revisit how to interpret the parameters of linear models in \Cref{sec:interpretLinear}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLengthBestFit.pdf}
\caption{Line of best fit between ratings and review length. \label{fig:ratingLengthBestFit}}
\end{SCfigure}

\paragraph{More Complex Models}

The above reasoning generalizes to fitting more complex models than a simple line, for example we could imagine that a rating could be related to both the length of the review and the number of comments the review received:
\begin{equation}
    \text{rating} \simeq \theta_0 + \theta_1 \times (\text{review\ length}) + \theta_2 \times (\text{n\_comments}).
    \label{eq:threeParameters}
\end{equation}
The above process---finding a line of best fit that best approximates the relationship between our observed features $X$ and labels $y$---describes the basic concept of \emph{linear regression}.

\paragraph{Adding More Dimensions}

Just as \Cref{eq:sysOfEq0} corresponded to fitting a line in two dimensions, \Cref{eq:threeParameters} now corresponds to fitting a plane in three. But ultimately the procedure for fitting this model remains the same. We simply have an additional column in our feature matrix:
\begin{equation}
    X = \left[
  \begin{tabular}{ccc} 1 & 2086 & 1 \\ 1 & 1521 & 1 \\ 1 & 1519 & 5 \\ 1 & 1791 & 1 \\ 1 & 1762 & 0 \\ \multicolumn{3}{c}{\vdots} \end{tabular}
  \right].
\end{equation}
Solving $\theta = (X^TX)^{-1}X^Ty$ yields
\begin{equation}
    \theta = \left[
    \begin{tabular}{c}
         3.954 \\
         $7.243 \times 10^{-5}$\\
         0.108
    \end{tabular} \right]
    \begin{tabular}{l}
      intercept\\
      slope for length\\
      slope for number of comments
    \end{tabular}.
    \label{eq:thetaThree}
\end{equation}
Interestingly, when we add this additional parameter $\theta_2$, the values of $\theta_1$ and $\theta_0$ are different from those of the model we previously fit (compare \Cref{eq:thetaThree,eq:thetaTwo}). Critically, the slope associated with the length term ($\theta_1$) is reduced in our new model. We discuss how to interpret these parameters in \Cref{sec:interpretLinear}.




\subsection{Regression in \emph{sklearn}}

Various libraries support the basic machine learning techniques described in this chapter, and indeed they can be implemented relatively straightforwardly via standard linear algebra operations. Here we describe the implementation in \emph{scikit-learn}
%\todo{style}
though other implementations follow similar interfaces.

First we load our dataset; here we read our sample (in this case a toy dataset of 100 reviews) in \emph{json} format,\footnote{\emph{Json} is a structured data format, made up of key-value pairs (where values can in turn be lists or other json objects). See \url{https://www.json.org/}.}
%\todo{italics}
which results in a list of 100 dictionaries:
%\done{json described anywhere? Added footnote}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
data = []
for l in open("fantasy_100.json"): # 100 reviews of fantasy novels from Goodreads
    d = json.loads(l)
    data.append(d)
\end{lstlisting}
\end{minipage}
%\done{Data or dataset? Looks fine}

Next we extract labels and features from the dataset. In this case we train a predictor to estimate ratings as a function of review length, as in \Cref{sec:?}:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
ratings = [d['rating'] for d in data] # The output we want to predict
lengths = [len(d['review_text']) for d in data] # The feature we'll use for prediction
\end{lstlisting}
\end{minipage}

To regress on this data we must first construct our matrix of features $X$ and our vector of labels $y$; note the inclusion of a constant feature in our feature matrix:\footnote{Although in fact this can be excluded and $\theta_0$ fit by the library by setting \texttt{fit\_intercept=True}; here we include it manually as a matter of best-practice.}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = numpy.matrix([[1,l] for l in lengths])
y = numpy.matrix(ratings).T
\end{lstlisting}
\end{minipage}

From here regressing is simply a matter of passing our features and labels to the appropriate model from \emph{sklearn}.
%\todo{italics}
%\todo{sklearn or scikit-learn}
Having done so we extract the coefficients $\theta$:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
model = sklearn.linear_model.LinearRegression(fit_intercept=False)
model.fit(X, y)
theta = model.coef_
\end{lstlisting}
\end{minipage}

Finally, we can confirm manually that the pseudoinverse from \Cref{eq:pseudoInv} yields the same result:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
numpy.linalg.inv(X.T*X)*X.T*y
\end{lstlisting}
\end{minipage}

(in both cases we find that $\theta = ?$).
%\todo{fill in correct value of theta}

\section{Evaluating Regression Models}
\label{sec:evaluateRegression}


When developing the linear models above, we were somewhat imprecise about what is meant by a `line of best fit' (or generally a model of best fit). Indeed, the pseudoinverse is not a `solution' to the system of equations given in \Cref{eq:sysOfEq2}, but is merely an approximation (naturally, the line of best fit does not pass through all points exactly).

Here, we would like to be more precise about what it means for a model to be `good'. This is a key issue when fitting and evaluating any machine learning model. One needs a way of quantifying how closely a model fits the given data. Given a desired measure of success, we can compare alternative models against this measure, and design optimization schemes that optimize the desired measure directly.

\subsection{The Mean Squared Error}
\label{sec:MSE}

A commonly used evaluation criterion when evaluating regression algorithms is called the Mean Squared Error, or MSE. The Mean Squared Error between a model $f_\theta(X)$ and a set of labels $y$ is defined as 
%\todo{make sure notation is consistent with rest of chapter}
\begin{equation}
    \text{MSE}(y, f_\theta(X)) = \frac{1}{|y|} \sum_{i=1}^{|y|} (f_\theta(x_i) - y_i)^2,
\end{equation}
in other words, the average squared difference between the model's predictions and the labels. Often reported is also the \emph{Root} Mean Squared Error (RMSE), i.e.,~$\sqrt{\text{MSE}(y, f_\theta(X))}$; the RMSE is sometimes preferable as it is consistent in scale with the original labels.
%\done{check this}

With some effort, it can be shown that the linear model $f_\theta(X)$ that minimizes the MSE compared to the labels $y$ is given by using the pseudoinverse as in \Cref{eq:pseudoInv}. We leave this as an exercise (\Cref{?}).
%\todo{write this exercise and reference}

\subsection{Why the Mean Squared Error?}
\label{sec:whyMSE}

Although the Mean Squared Error has a convenient relationship with the pseudoinverse, it may otherwise seem a somewhat arbitrary choice of error measure. For instance, it may seem more obvious at first to compute an error measure such as the Mean Absolute Error (or MAE):
\begin{equation}
    \text{MAE}(y, f_\theta(X)) = \frac{1}{|y|} \sum_{i=1}^{|y|} | f_\theta(x_i) - y_i |
    \label{eq:MAE}
\end{equation}
Or, why not count the number of times the model is wrong by more than one star? Or why not measure the mean \emph{cubed} error?

To defend the mean squared error as a reasonable choice, we need to characterize what types of errors are more `likely' than others. Essentially, the Mean Squared Error is assigning small penalties to small errors, and \emph{very} large penalties to large errors. This is in contrast to, say, the Mean Absolute Error, which assigns penalties exactly proportional to how large the error is. What the Mean Squared Error therefore seems to be assuming is that small errors are \emph{common} and large errors are very uncommon.

What we are talking about informally above is a notion of how errors are \emph{distributed} under some model. Formally, we say that the labels are equal to our model's predictions, plus some error:
\begin{equation}
    y = \underbrace{f_\theta(X)}_{\text{prediction}} + \underbrace{\epsilon}_{\text{error}},
\end{equation}
and that our error follows some probability distribution. Our argument above said that small errors are common and large errors are very rare. This suggests that errors may be distributed following a \emph{bell curve}, which we could capture with a Gaussian (or `Normal') distribution:
\begin{equation}
    \epsilon \sim \mathcal N(0, \sigma^2).
\end{equation}
The density function for a (zero mean) Gaussian distribution is given by
\begin{equation}
    f'(x') = \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{x'}{\sigma}\right)^2}
\end{equation}
(we use the notation $f'$ and $x'$ to avoid confusion with $f$ and $x$ elsewhere). So, the probability density of an error as large as $y_i - f_\theta(x)$ is given by
\begin{equation}
    \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y - f_\theta(x)}{\sigma}\right)^2}.
    \label{eq:errorDensity}
\end{equation}
This density function is depicted in \Cref{fig:gaussianDensity}.
%\done{squash plot so that it's same as others? It's fine, at least one other is the same}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/gaussianError.pdf}
\caption{Gaussian error density. \label{fig:gaussianDensity}}
\end{SCfigure}

\subsection{Maximum Likelihood Estimation of Model Parameters}
%\todo{subsection or subsubsection? organization could be better. or could add some gluing text above}

%Now that we have this type of density function in mind, we can begin to reason more formally about what it means for a particular model to be a `good' fit to the data.
Having defined the density function above, we can now reason more formally about about what it means for a particular model to be a `good' fit to the data. In other words, we would like to ask how `likely' a particular model is in terms of a given error distribution. %For example, are the errors we observe consistent with a bell curve shape? 
%\todo{cut this text up a lot, check}

Specifically, the density function in \Cref{eq:errorDensity} gives us a means of assigning a probability (or likelihood) to a particular set of labels $y$, given features $X$ and a model $\theta$, under some particular error distribution (in this case a Gaussian):
%\done{mouthful. Fine}
\begin{equation}
    \mathcal L_\theta(X | y) = \prod_{i = 1}^{|y|} \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}.
    \label{eq:likelihood1}
\end{equation}
Essentially, we want to choose $\theta$ so as to maximize this likelihood.
% Something else here?
Our goal intuitively is to choose a value of $\theta$ that is consistent with this error distribution, i.e.,~a model that makes many small errors and few large ones.

Precisely, we would like to find
%\begin{equation}
    $\argmax_{\theta} \mathcal L_\theta(X | y)$.
%\end{equation}
This procedure (finding a model $\theta$ that maximizes the likelihood under some error distribution is known as \emph{Maximum Likelihood Estimation}.
We solve by taking logarithms and removing irrelevant terms ($\pi$, $\sigma$):
\begin{align}
  \argmax_\theta \mathcal L_\theta(X | y) & = \argmax_\theta \ell_\theta(X | y)\\
  & = \argmax_\theta\log \prod_{i = 1}^{|y|} \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}\\
  & = \argmax_\theta \sum_i \log e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}\\
  & = \argmax_\theta -\sum_i (y_i - f_\theta(x_i))^2\\
  & = \argmin_\theta \sum_i (y_i - f_\theta(x_i))^2\\
  & = \argmin_\theta \frac{1}{|y|} \sum_i (y_i - f_\theta(x_i))^2
  \label{eq:MSEMLEderiv}
\end{align}
Note crucially in the above equation that \emph{the maximum likelihood solution for $\theta$ under our Gaussian error model is precisely the Mean Squared Error}. This demonstrates the relationship between the Mean Squared Error and Maximum Likelihood Estimation (which we summarize in 
%\Cref{box:MSEMLE}).
Box~\ref{box:MSEMLE}).
%\todo{text here, and in the box, is a bit clunky, try rewriting}

\begin{figure}
\caption{The MSE and the MLE}
The argument we made in \Cref{sec:whyMSE} explained our motivation behind the choice of the Mean Squared Error (MSE): by choosing the MSE as our error metric, we are implicitly assuming that our model's errors follow a Gaussian distribution. This assumption is explained by the fact that minimizing the mean squared error maximizes the likelihood of the observed errors under a Gaussian error model.
\label{box:MSEMLE}
\end{figure}

The above arguments may seem like just a mathematical curiosity, and indeed in practice we will often minimize the Mean Squared Error without scrutinizing the decision to do so. But this relationship between error functions and probabilities will come up regularly when we develop models for classification (\Cref{sec:classification}), sequence mining (\Cref{chap:temporalSequential}), and recommender systems (\Cref{chap:recsys}). To summarize a few key points:
\begin{enumerate}
    \item When we optimize a certain error criterion, we are often making implicit assumptions about how errors are distributed.
    \item Sometimes, a model will poorly fit a dataset because these assumptions are violated. Understanding the underlying assumptions gives us a chance to diagnose them and attempt to correct it (\Cref{subsec:errorsArent}).
    %\done{below is paragraph, so can't be referenced. Is subsection}
    \item In many of the models we fit later (including when we develop classifiers in \Cref{sec:?}), 
    %\done{which? Done} 
    we will use this style of probabilistic language, i.e., we will talk about some observed data having high \emph{likelihood} under some model. Fitting such models will use this same strategy of selecting a model which maximizes this likelihood.
\end{enumerate}

\subsection{The $R^2$ Coefficient}

Having motivated our choice of the Mean Squared Error at some length, it is worth asking how low the MSE should be before we consider our model to be `good enough'?

This quantity turns out not to be well-defined: the Mean Squared Error will depend on the scale and variability of our data, and the difficulty of our task. For example, predicted ratings on a 5-point scale would likely have lower MSEs than predicted ratings on a 100-point scale; on the other hand, this might not be the case if ratings on a 100-point scale were highly concentrated (e.g.~nearly all ratings were in the 92-95 range). Finally, the MSE in either setting could be higher simply due to a lack of available features that allow us to predict ratings accurately.

As such, we would like a calibrated measurement of model error. As we just argued, the MSE is related to the \emph{variance} of the data: this relationship is easy to see as follows:
%\todo{Why is this MSE(y|X) and not MSE(f|y,X)?}
\begin{align}
    \bar{y} & = \frac{1}{|y|} \sum_i y_i\\
    \mathrm{var}(y) & = \frac{1}{|y|} \sum_i (y_i - \bar{y})^2\\
    \mathrm{MSE}(y|X) & = \frac{1}{N} \sum_i (y_i - f(x_i))^2
\end{align}
In other words, the Mean Squared Error would be equal to the variance if we had a trivial predictor that always estimated $f(x_i) = \bar{y}$.\footnote{Note that this is the best we could do if using a trivial predictor of the form $f(x_i) = \theta_0$ (\Cref{ex:?}).} Thus the variance might be used as a way of normalizing the Mean Squared Error:
\begin{equation}
    \mathrm{FVU}(y|X) = \frac{\mathrm{MSE}(y|X)}{\mathrm{var}(y)}
\end{equation}
This quantity, known as the \emph{Fraction of Variance Unexplained} essentially measures the extent to which the model \emph{explains variability} in the data, as compared to a predictor which always predicts the mean (i.e.,~one which explains no variability at all).

This quantity will now take a value between 0 and 1: 0 being a perfect classifier (MSE of zero), and 1 being a trivial classifier.\footnote{The FVU \emph{could} be greater than 1, if our classifier were \emph{worse} than a trivial one.}

Often, one reports the \emph{$R^2$ coefficient}, which is simply one minus the FVU:
\begin{equation}
    R^2 = 1 - \frac{\mathrm{MSE}(y|X)}{\mathrm{var}(y)},
\end{equation}
which now takes a value of 1 for a perfect predictor, and 0 for a trivial predictor. The name `$R^2$' comes from a different way of deriving the same quantity, in terms of the correlation between the predictions and the labels.\footnote{We omit this alternate derivation for now, but revisit the idea of correlation briefly in \Cref{sec:pearsonSim}.}



\subsection{What to do if Errors \emph{Aren't} Normally Distributed?}
\label{subsec:errorsArent}

The arguments we made above characterized the relationship between the Mean Squared Error and the normal (Gaussian) distribution. In summary, the MSE 
%\todo{consistently abbreviate} 
is a reasonable choice so long as our model errors are expected to be centered around zero, and not to have large outliers.

But what can we do if these assumptions do not hold? First, we consider how to validate the assumptions in the first place. Recall that our assumptions assume that the residuals,
\begin{equation}
    r_i = (y_i - f_\theta(x_i))
    \label{eq:qq}
\end{equation}
follow a normal distribution. To begin with, a simple plot may reveal whether the residuals follow the desired overall trend.

\Cref{fig:qq} (left) shows a histogram of residuals $r_i$ for a simple prediction task, in which we estimate review lengths as a function of user gender (covered later in \Cref{sec:binaryFeat}).
Although the plot has a slight bell shape, it deviates from the normal distribution in several key ways, for instance:
\begin{itemize}
    \item The residuals do not appear to be centered around zero. In fact the \emph{average} residual is zero,\footnote{In fact, the average residual of this type of linear regression model is \emph{always} zero (see \Cref{exercise:?}).} though largest bins in the histogram are somewhat below zero.
    \item The are some large outliers (i.e.,~extremely long reviews whose length was underpredicted).
    \item There are no small outliers, and there is almost no `left tail,' i.e.,~the model never significantly overpredicts.
\end{itemize}

\begin{figure}
\begin{center}
\includegraphics[scale=0.75]{fig/qqhist.pdf}
\includegraphics[scale=0.75]{fig/qq.pdf}
\end{center}
\caption{Histogram of observed residuals (left), and residuals compared to theoretical quantiles under a normal distribution. \label{fig:qq}}
\end{figure}

Although the histogram in \Cref{fig:qq} allows us to quickly assess whether the residuals follow a normal distribution, this can be visualized more precisely by comparing the theoretical quantiles of a normal distribution to the observed residuals, as in \Cref{fig:qq}, right.\footnote{This type of diagnostic plot can be generated easily with a library function, e.g.~this one was generated with \texttt{scipy.stats.probplot}.} The plot essentially compares the (sorted) residuals to those we would expect if we were to sample the same number of values from a normal distribution: if our residuals followed a normal distribution, plotting these quantities against each other would result in a straight line. Again, the plot basically reveals that there is an unusual outlier, and that residuals are missing the left tail (i.e., overpredictions) that would be expected. Note that this same type of diagnostic tool can be used to compare our residuals against any distribution in the same way.

While the above is merely a diagnostic for determining \emph{whether} the residuals followed a normal distribution, the more difficult question is how these discrepancies can be corrected. Some general guidelines are as follows:
\begin{description}
    \item[Remove outliers] The normal distribution (and thus the MSE) is especially sensitive to outliers due to how it penalizes large errors. To the extent that extremely long reviews do not conform to the usual behavior of the data, we could simply discard them before training.
    \item[Choose an error distribution less sensitive to outliers] For example, the Mean Absolute Error assigns a smaller penalty to large mispredictions, so outliers will have a smaller effect on the model.
    \item[Choose a skewed distribution] In this example we are predicting lengths, which by definition are bounded below (at length zero) but not above. Thus there will be a long tail of underpredctions, but not large overpredictions. We might account for this by modeling the data using a skewed probability distribution (such as a Gamma distribution.
    \item[Fit a better model] Note that the diagnostic in \Cref{fig:qq} is a function of the \emph{errors}, rather than the original data. Thus, for example, if we had a feature that allowed us to correctly predict the length of the unusually long review, the errors may again become more consistent with a normal distribution.
\end{description}

Again, most of the time the Mean Squared Error is a safe and reasonable choice, and can be used without too much scrutiny.
%\todo{Feel like I wrote this text elsewhere}
Nevertheless it is useful to have a sense of its underlying assumptions so that one can detect when they have been violated.

\section{Feature Engineering}

Along with the simple linear function relating features to labels as in \Cref{eq:?} come significant limitations in terms of what kinds of relationships can be modeled with linear regression techniques. 
%For instance, when trying to model features like gender (categorical), temporal information (periodic)
When trying to model asymptotic, periodic, or other non-linear relationships between features and labels, it is not yet clear how this can be accomplished given the limitations of this type of model.

As we shall see, complex relationships can be handled within the framework of linear models, so long as we exercise care by appropriately transforming our features (and labels). In practice, the success or failure of our models will often depend on carefully processing our data to help the model uncover the most salient relationships.  This process of \emph{feature engineering} proves critical even when developing deep learning models based on images or text: in spite of the vague promise of learning complex non-linear relationships automatically, extracting meaningful signals from data is often a matter of careful engineering, rather than selecting a more complex model.

\subsection{Simple Feature Transformations}
\label{sec:simpleTransforms}

The first model we fit in \Cref{eq:sysOfEq0} revealed a positive association between review length and ratings. However, fitting the data with a line (\Cref{fig:ratingLengthBestFit}) does not seem to fit the data very accurately. Fitting the data with a line seems limiting, given that the trend may be better captured by a polynomial or asymptotic function (since the rating can not grow above five stars).

Na\"ively, we might think that this is a fundamental limitation of linear models. Note however that the assumption of linearity in $\theta$ (\Cref{eq:linearTheta}) does not prevent us from fitting (for example) a polynomial function. The polynomial equation
\begin{equation}
  \text{rating} \simeq \theta_0 + \theta_1 \times (\text{review\ length}) + \theta_2 \times (\text{review\ length})^2
  \label{eq:sysOfEqPoly}
\end{equation}
is linear in $\theta$, even though we have transformed the input features in $X$.

This idea can be applied straightforwardly to fit polynomial functions, as shown in \Cref{fig:cubicFit}.\footnote{Actually, these curves were generated using the feature $\frac{\text{length}}{1000}$, as the matrix inverse $(X^TX)^{-1}$ becomes numerically unstable given large values of $\text{length}^3$.}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLengthBestFitQuad.pdf}
\caption{Quadratic and cubic polynomials of best fit. \label{fig:cubicFit}}
\end{SCfigure}

\subsection{Binary and Categorical Features: One-Hot Encodings}
\label{sec:binaryFeat}

So far we have dealt with regression problems where we have both real-valued inputs (features $X$), and real-valued outputs (labels $y$). What can we do in cases where features are binary or categorical?

As an example, let's consider whether the length of a user's review can be predicted by (or more simply, is related to) their gender. To do so, we'll look at a different dataset (of a few hundred beer reviews) that includes the gender of its users.

That is, we'd like a model of the form:
\begin{equation}
    \text{length} \simeq \theta_0 + \theta_1 \times \text{gender}.
    \label{eq:genderModel}
\end{equation}
Obviously, gender (represented in this dataset as a string) is not a numerical quantity, so we need some appropriate \emph{encoding} of the gender variable.

For the moment, let's treat gender as a binary variable. We'll relax this assumption in a moment to allow for a non-binary gender variable (and allow for the possibility that the gender is missing, as it can be in this dataset), but for the moment let's encode the gender variable as:
\begin{equation}
    \text{Male} = 0; \quad \text{Female} = 1.
    \label{eq:genderEncoding1}
\end{equation}
Alternately, this is just a binary indicator specifying whether this user is female. This encoding, although only one of a few we might have used, allows us to fit a linear model and estimate the values of $\theta_0$ and $\theta_1$. The model we fit (after removing users who did not specify a gender) is
\begin{equation}
    \text{length (in words)} \simeq 127.07 + 8.76 \times ( \text{user is female} ).
\end{equation}
With a little thought, we can interpret the model parameters as indicating that, on average, females write slightly longer reviews (by 8.76 words) compared to males. Note that 127.07 is not the population average, but rather the average for males (whose gender feature is zero); we will revisit how to interpret the parameters of these models in \Cref{sec:interpretLinear}.

A scatter plot of the data (i.e., the encoded gender attribute and the review lengths), as well as the line of best fit above is depicted in \Cref{fig:genderLength}. Note that although we have fit the data with a line (\Cref{fig:genderLength}, left), the actual feature values only occupy two points (0 and 1); thus the fit is perhaps better represented with a bar plot (\Cref{fig:genderLength}, right). 
%Note the fact that the model would predict (for example) a length of 1,000 words if the gender feature took a value of 100 does not invalidate it, as such values are never observed.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/genderVlength.pdf}
\includegraphics[scale=0.75]{fig/genderVlengthBar.pdf}
\caption{Gender versus review length (beer data). Visualized via a line of best fit (left) and a bar plot (right). \label{fig:genderLength}}
\end{SCfigure}

\subsubsection{Categorical Features}
%\todo{Replace various paragraphs with subsubsections}

%\todo{Probably this should be a subsection}
In practice, the gender attribute may assume more than binary labels in some datasets. To accommodate this, we might na\"ively imagine extending our encoding from \Cref{eq:genderEncoding1} to include additional values:
%\todo{Four category gender figure shouldn't have axis label `gender?'}
\begin{equation}
\begin{tabular}{ll}
    Male = & 0;\\
    Female = & 1;\\
    Other = & 2;\\
    Not specified = &3;\\
    etc.
    \end{tabular}
    \label{eq:oneHot1}
\end{equation}
Again we fit the same model as in \Cref{eq:genderModel}. Doing so we might obtain a fitted model like the one in \Cref{fig:genderFit4} (left).

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/genderVlengthBar4.pdf}
\includegraphics[scale=0.75]{fig/genderVlengthBar4oneHot.pdf}
\caption{Categorical features with a na\"ive sequential encoding (left), and a one-hot encoding (right). \label{fig:genderFit4}}
\end{SCfigure}

Note that the model fit in \Cref{fig:genderFit4} (left) implicitly makes some dubious assumptions. For example, because the model is linear, it assumes that the difference between `male' and `female' lengths is \emph{the same as} the difference between `female' and `other' lengths.
\footnote{That is, males receive the prediction $\theta_0$; females receive $\theta_0 + \theta_1$; other receives $\theta_0 + 2\theta_1$, etc.}

This assumption is not supported by the data, and in fact would be different if we simply reordered our indices in \Cref{eq:oneHot1}. Rather, we would like to associate different predictions to members of each group, as in \Cref{fig:genderFit4} (right). This can be achieved via a different encoding:
\begin{equation}
    \begin{tabular}{lll}
      Male = & $[0,0,0]$\\
      Female = & $[0,0,1]$\\
      Other = & $[0,1,0]$\\
      Not specified = & $[1,0,0]$\\
      etc.
    \end{tabular}
    \label{eq:oneHot2}
\end{equation}
%Using the above encoding, 

We can quickly confirm that the model would make predictions as follows:
\begin{equation}
    \begin{tabular}{ll}
    Male: & $y = \theta_0$\\
    Female: & $y = \theta_0 + \theta_1$\\
    Other & $y = \theta_0 + \theta_2$\\
    Not specified & $y = \theta_0 + \theta_3$\\
    etc.
    \end{tabular}
\end{equation}
That is, $\theta_0$ is the prediction for males, $\theta_1$ is the \emph{difference} between females and males, etc. Note that we now have four parameters to estimate four values, as opposed to two parameters as in \Cref{eq:oneHot1}. As such, the model has sufficient flexibility to make different estimates for each group, as in \Cref{fig:genderFit4} (right).

This type of encoding, in which we have a separate feature dimension for each category, is called a \emph{one-hot} encoding. 
%\done{is this the first time I mention? Yes}

Note that to represent four categories in \Cref{eq:oneHot2} we only used \emph{three}-dimensional features (or in general, for $N$ categories, we could use an $(N-1)$-dimensional encoding. Possibly this seems slightly confusing compared to using a four-dimensional feature vector (e.g.~$\text{Male} = [0,0,0,1]$, etc.). Two reasons for using an $(N-1)$ dimensional feature vector are as follows:
\begin{itemize}
  \item Doing so is not necessary; together with $\theta_0$, the representation in \Cref{eq:oneHot2} uses four parameters to predict four values, so adding an additional dimension would add no more expressive power to the model and would be redundant.
  \item Doing so could possibly be harmful. While adding a redundant feature seems harmless, in practice it means the system of equations in \Cref{eq:?} would no longer have a unique solution, as the matrix $X^TX$ would be uninvertable (\Cref{exercise:?}).
\end{itemize}

Similarly, a \emph{multi-hot} encoding can be used in cases where an instance can belong to multiple categories simultaneously, for example for a `race' feature, a user may associate with multiple racial groups.
%\todo{crappy example}
Note that this is equivalent to a concatenation of several binary features.

\subsection{Missing Features}
\label{sec:missingFeat}

Often datasets will have features that are missing, for example the underlying data used for the example in \Cref{sec:binaryFeat} consisted of a gender attribute that many users may leave unspecified.

When dealing with binary or categorical features we dealt with these missing values quite straightforwardly---we simply treated `missing' as an additional category.

But if a continuous feature, such as a user's age or income, were missing, we must think harder about how to handle it. Trivially, we might simply discard instances with missing features, though this strategy will harm model performance if it means discarding a substantial fraction of our data.

Alternately we might replace the missing entries by the average (or mode) value for that feature; this strategy is known as `feature imputation.' This may be more effective than discarding the feature, but may also introduce some bias, as (for example) users who choose to leave a feature unspecified may be quite different from the average or mode.

To avoid the above issues, we would like a strategy that uses features when they are available, but makes separate predictions for those users when they are not. This can be achieved via the following strategy: for any feature $x$ which is sometimes missing, replace it by two features $x'$ and $x''$ as follows:
\begin{equation}
    x' = \left \lbrace
    \begin{tabular}{ll}
      1 & if feature is missing\\
      0 & otherwise
    \end{tabular} \right. , \quad
    x'' = \left\lbrace
    \begin{tabular}{ll}
      0 & if feature is missing\\
      $x$ & otherwise
    \end{tabular} \right. .
\end{equation}
Following this parameters can be fit within a model as usual:
\begin{equation}
    y = \theta_0 + \theta_1 x' + \theta_2 x''.
\end{equation}
The above representation may seem somewhat arbitrary, but makes sense once we expand the expression for missing and non-missing features. E.g.~when a feature is available predictions are made according to
\begin{equation}
    y = \theta_0 + \theta_2 x,
\end{equation}
whereas when a feature is missing predictions are made according to
\begin{equation}
    y = \theta_0 + \theta_1.
\end{equation}
This achieves the desired effect: when the feature is available we predict as normal, and when the feature is unavailable we predict using a learned value ($\theta_1$). Note that this strategy is very similar to feature imputation, but rather than using a heuristic imputation strategy, the model will directly learn what is the best prediction to impute.

\subsection{Temporal Features}
\label{sec:temporalFeatures}

Temporal features may make excellent predictors in various settings. Outcomes such as ratings, clicks, purchases (etc.)~are often influenced by factors such as the day of the week, the season, or long-term trends that span several years.

Let's explore an example in which we try to predict the rating of a book on Goodreads based on the day of the week that it was entered. Average ratings for each weekday\footnote{Again based on a small sample of reviews from the Fantasy genre.} are shown in \Cref{fig:temporalFit}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/timePlot.pdf}
\caption{Ratings as a function of the weekday, and line of best fit. \label{fig:temporalFit}}
\end{SCfigure}

As before, we might try to describe this relationship using a line, i.e.,~to fit a model of the form 
%\todo{make sure use of $=$ and $\simeq$ is consistent throughout this section}
\begin{equation}
    \text{rating} = \theta_0 + \theta_1 \times (\text{day of week}).
    \label{eq:fitDoW}
\end{equation}
For this equation to make sense, we need to map the day of the week to a numeric quantity. A trivial encoding might assign numbers sequentially, e.g.
\begin{equation}
    \text{Sunday} = 1; \quad \text{Monday} = 2; \quad \text{Wednesday} = 3; \quad \text{etc.}.
\end{equation}
Fitting \Cref{eq:fitDoW} using this representation yields the line of best fit depicted in \Cref{fig:temporalFit}, which reveals a slight upward trend as the days of the week progress.

The linear trend in \Cref{fig:temporalFit} seems a fairly poor fit to the data; we might think about fitting more complex functions like polynomials to better capture the observed data. But consider that our model is essentially periodic: Sunday (represented by a 1) follows Saturday (represented by a 7), though we could just as easily have represented Wednesday as 1 and Tuesday as 7. These choices seem arbitrary, but they impact our model in unexpected ways.

The above point is perhaps clearer if we visualize our model's predictions over a period of two weeks, as in \Cref{fig:periodic}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/timePlotPeriodic.pdf}
\caption{If we consider that our weekly measurements are periodic, we realize that fitting periodic data with a linear trend seems unrealistic. \label{fig:periodic}}
\end{SCfigure}

We'll revisit the critical role of temporal features in the context of recommendation in \Cref{sec:caseStudyNetflix}, and in 
%\todo{check this, add others}

\subsection{Transformation of Output Variables}

Finally, just as we saw how to transform features in \Cref{sec:simpleTransforms}, we can also transform our \emph{output} variables.

For example, let's consider fitting a model to determine whether resubmitted posts on \emph{reddit}
%\todo{Cite datasets somehow?}
receive lower numbers of upvotes, i.e.,
%long reviews on \emph{Steam} tend to be associated with games with a long play time, i.e.,
% \begin{equation}
%     \text{time played (hours)} = \theta_0 + \theta_1 (\text{review length}).
% \end{equation}
\begin{equation}
    \text{upvotes} = \theta_0 + \theta_1 \times (\text{submission number})
    \label{eq:reddit1}
\end{equation}
(where the `submission number' is `1' for an original resubmission, `2' for the first resubmission, etc.)
This model, along with the observations on which it is based, are shown in \Cref{fig:redditResub} (left).

% \begin{figure}
%     \centering
%     \includegraphics[scale=0.75]{fig/lengthPlayed.pdf}
%     \includegraphics[scale=0.75]{fig/lengthPlayedLog.pdf}
%     \caption{Review length versus hours played (left), and logarithm of hours played (right), from \emph{Steam} video game reviews. Lines of best fit for both plots are included.}
%     \label{fig:lengthPlayed}
% \end{figure}

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/redditUp.pdf}
    \includegraphics[scale=0.75]{fig/redditUpLog.pdf}
    \caption{Number of upvotes versus submission number on \emph{reddit}. The left plot shows the original data (with averaged upvote counts), the right plot shows the logarithm of the number of upvotes. Lines of best fit for both plots are included.}
    \label{fig:redditResub}
\end{figure}

Although the line of best fit indicates a slight downward trend, it does not appear to correspond closely to the overall shape of the data. 
%Part of the issue is that the data contains numerous \emph{outliers} (i.e.,~games with extremely large `time played' values), that the model is unable to reliably predict; as we discussed in \Cref{sec:whyMSE}, such large outliers do not fit the assumptions made when taking our line of best fit, and ultimately spoil the model's ability to capture a meaningful trend.
%\todo{Write this, or maybe not}
Possibly we might imagine that the data follows an exponentially decreasing trend, e.g.~every time you resubmit a post, you can expect to receive half as many upvotes.

Again, one might assume that this type of trend is something that cannot be captured by a linear model. But in fact we can possibly address this by transforming the output variable $y$. For example, consider fitting
\begin{equation}
    \log_2(\text{upvotes}) = \theta'_0 + \theta'_1 (\text{submission number})
    \label{eq:reddit2}
\end{equation}
%(note that we add a constant to ensure that the argument of the logarithm is always positive).
Now, a unit change in the prediction corresponds to a post receiving \emph{twice} as many upvotes.
%a game being played \emph{twice} as long. 
While this is still a linear model, the model corresponds to fitting
\begin{equation}
    \text{upvotes} = 2^{\theta'_0 + \theta'_1 (\text{submission number})}.
    \label{eq:reddit3}
\end{equation}
The transformed data and line of best fit are shown in \Cref{fig:redditResub} (right). 

Arguably, this second line better captures the overall trend, and does not have the same issues with outliers. 
%
In fact, if we transform the fitted values from \Cref{eq:reddit2} back to their original scale via \Cref{eq:reddit3}, the transformed values actually have a Mean Squared Error about 10\% \emph{lower} than the model from \Cref{eq:reddit1}, indicating that the transformed data more closely follows a linear trend compared to the untransformed data.
%\todo{mouthful}
%compute the Mean Squared Error of the models in \Cref{eq:reddit1} and \Cref{eq:reddit2}, the model from \Cref{eq:reddit3} has 



\section{Interpreting the Parameters of Linear Models}
\label{sec:interpretLinear}

When analyzing the linear models developed so far, we have already talked about interpreting their parameters in terms of general trends, correlation, differences between groups, etc.

While is tempting to casually interpret the meaning of various features, we must be careful and precise about doing so.

First, we should be precise about the interpretation of our slope and intercept terms. For example, when we modeled rating as a function of review length (\Cref{eq:reviewLength1}), we stated that under our model, ratings increased fractionally ($1.194\times 10^{-4}$) for every character of a review.

This interpretation makes sense given a model containing only a single features, but as soon as we incorporate multiple features we must be more careful. Consider e.g.~the model from \Cref{eq:thetaThree}, in which we included both the length and number of comments as predictors. We could no longer state that under this model, that the rating increases (by $7.243 \times 10^{-5}$) for every character in the review. Precisely, we must interpret the parameters as follows: \emph{Our prediction of the rating increases by $7.243\times 10^{-5}$ for every character in the review, assuming the other features remain unchanged}. This definition is stated precisely in \Cref{box:interpretLinear}.

\begin{figure}
\caption{Interpreting the parameters of linear models \label{box:interpretLinear}}
%\todo-{Capitalize captions in boxes or not?}
Given a linear model $y = X\theta$ we should interpret a parameter $\theta_k$ as follows:

\emph{For every unit change in $x_{ik}$, our prediction of the output $y_i$ would increase by $\theta_k$ \emph{if all other features remain fixed}}

It is important to note that we are talking about the model's \emph{prediction} (rather than an actual change in the label), which could change if different features were included. And we must include the condition that other features remain constant. Without which we would fail to account for the correlations among different features.
\end{figure}

Critically, features like review length and number of comments may be highly \emph{correlated}, such that we would rarely see longer reviews without also seeing more comments. For example, when incorporating features based on polynomial function (as in \Cref{eq:sysOfEqPoly}), or when dealing with one-hot encodings (as in \Cref{eq:oneHot1}), a feature generally \emph{cannot} change without the other features changing.

%\todo{clean up writing a little}
Second, we should be clear when interpreting parameters that we are talking about \emph{predictions} under a particular model rather than actual changes in the output variable $y_i$. These predictions can change as we include additional features, and a feature that had previously been predictive may be less predictive in the presence of another (as we saw in \Cref{eq:thetaThree}); likewise, we should be careful not to conclude that (e.g.)~length is \emph{not} correlated with the output, simply because another feature is \emph{better} correlated.

Finally, we should be careful not to make statements about the \emph{causal} effect of features on the output variable. Our line of best fit does not state that long reviews `cause' positive opinions any more than it states that positive opinions cause long reviews.

% Second, we should note that the 
% Consider a trivial example in which we predict a student's GPA as a function of their commute time:
% \begin{equation}
%     \mathrm{GPA} = 3.1 - 0.15 \times (\text{commute time to school in hours}).
%     %\todo{Could try a real example}
% \end{equation}
% From this one might casually conclude that long commutes contribute to lower GPAs. However if we fit a model that includes a third feature, hours employed, we might instead fit
% \begin{equation}
%     \mathrm{GPA} = 3.2 - 0.02 \times (\text{commute time}) - 0.08 \times (\text{hours worked})
% \end{equation}


\section{Fitting Models with Gradient Descent}
\label{sec:gradient}

When solving regression problems in the previous section, we looked for \emph{closed form} solutions. That is, we set up a system of equations (\Cref{eq:linearTheta}) in $\theta$, and attempted to solve them for $\theta$ (albeit approximately via the pseudoinverse).

As we begin to fit more complex models
%\todo{where does regularization go?} 
including in \Cref{sec:classification,sec:regularization}, a closed-form solution may no longer be available.

\emph{Gradient descent} is an approach to search for the minimum value of a function, by iteratively finding better solutions based on an initial starting point. The process (depicted in \Cref{fig:gradient}) operates as follows: 
%\todo{algorithmic environment}
\begin{enumerate}
    \item Start with an initial guess for $\theta$
    \item Compute the derivative $\frac{\partial}{\partial \theta}f(\theta)$. Here $f(\theta)$ is the MSE (or whatever criterion we are optimizing) under our model $\theta$.
    \label{step2}
    \item Update our estimate of $\theta \coloneqq \theta - \alpha \cdot f'(\theta)$
    \label{step3}
    \item Repeat Steps \ref{step2} and \ref{step3} until convergence.
    \label{step4}
\end{enumerate}
During each iteration, the process now follows the path of steepest descent, and will gradually arrive at a minimum of the function $f_\theta$.\footnote{Assuming the function is `well-behaved,' e.g.~the objective is bounded below, and the function is differentiable everywhere; though these are rarely issues when dealing with error functions specifically.}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/gradient.pdf}
%\todo{Figure has problems (e.g. f(heta)), and annotation looks funny}
\caption{Gradient descent demonstration. \label{fig:gradient}}
\end{SCfigure}

The above is a simple description of the procedure that omits many details. We omit some of the specifics for now and save them until we implement the procedure in practice, 
%\todo{pick some way to reference code from the text}
though some of the main issues include:
\begin{itemize}
    \item Given the starting point in \Cref{fig:gradient}, the algorithm would only achieve a \emph{local} rather than a \emph{global} optimum. To address this we could investigate ways to come up with a better initial `guess' of $\theta$, or later,
    %\todo{mentioned later?} 
    we could investigate alternative optimization approaches less susceptible to local minima.
    \item The step size $\alpha$ (Step~\ref{step3} above) must be chosen carefully. If $\alpha$ is too small, the procedure will converge very slowly; if $\alpha$ is too large, the procedure may `overshoot' the minimum value and obtain a \emph{worse} solution during the next iteration. Again, other than carefully tuning this parameter, we could investigate optimization methods not dependent on choosing this rate (see e.g.~quasi-Newton methods such as L-BFGS \cite{liu1989limited}).
    \item `Convergence' as defined in Step~\ref{step4} is not well-defined. We might define convergence in terms of the change in $\theta$ (or $f_\theta(X)$) during two successive iterations, or alternately we may terminate the algorithm once we stop making progress on held-out (validation) data, as in \Cref{sec:validation}.
    %\todo{validation isn't a section, edit a little}
\end{itemize}

\subsection{Linear Regression via Gradient Descent}

To solidify the ideas above, let's consider the specific example of minimizing the Mean Squared Error of a linear model, i.e.,
%\todo{Consistent way of writing $(y - x_i \cdot \theta)$, e.g. does $y$ go first. Compare to $R^2$ equation}
\begin{equation}
    f_\theta(y;X) = \frac{1}{|y|} \sum_{i = 1}^{|y|} (x_i \cdot \theta - y_i)^2.
\end{equation}
The
%\todo{equation above has both $X$ and $y$. This should be corrected everywhere}
derivative $f'(\theta)$ can be computed as follows:
%\todo{Check that I'm writing $X$ when I mean $X$ and $x_i$ when I mean $x_i$}
\begin{equation}
   \frac{\partial f}{\partial \theta_k} = \frac{1}{|y|} \sum_{i = 1}^{|y|} 2 x_{ik} (x_i \cdot \theta - y_i).
\end{equation}
Note that the above is a \emph{partial} derivative in $\theta_k$, which must be computed for each feature dimension $k = \lbrace 1 \ldots K \rbrace$.
%\todo{put in notation table}
The above derivative is more obvious after expanding $x_i \cdot \theta = \sum_{k=1}^K x_{i,k}\theta_k$.



\section{Non-linear Regression}
\label{sec:nonlinear}

So far, we have limited our discussion to models of the form $y = X\theta$, mostly because these offered us a convenient (closed form) solution to finding lines of best fit in terms of $\theta$.

However, this type of model has several limitations that we might wish to overcome, such as:
\begin{itemize}
    \item We cannot incorporate simple constraints on our parameters, such as that a certain parameter should be positive, or that one parameter is larger than another (which might be based on domain knowledge of a certain problem).
    \item Although we can manually engineer non-linear transforms of our features (as we did in \Cref{sec:simpleTransforms}, we cannot have the model learn these non-linear relationships automatically.
    \item The model cannot learn complex \emph{interactions} among features, e.g.~that length is correlated with ratings, but only if the user is female.\footnote{To be precise, the linear model does consider relationships among features in the limited sense that parameters for one feature will change in the presence of other correlated features (\Cref{sec:interpretLinear}); and, the model \emph{could} capture relationships between (e.g.)~gender and length if we were to manually engineer a feature describing this relationship. Our point here is about whether the model can learn these relationships automatically.}
\end{itemize}

The above goals can potentially be realized if we are allowed to transform model \emph{parameters}: for instance, we could ensure that a particular parameter was always positive by fitting
\begin{equation}
    \theta_k = \log(1 + e^{\theta_k'})
\end{equation}
(this is known as a `softplus' function; note that this function smoothly maps $\theta_k' \in \mathbb R$ to $\theta_k \in (0,\infty)$); or if we wanted one feature to be larger than another (e.g.~$\theta_k > \theta_j$) we could simply add the positive quantity above to another feature:
\begin{equation}
    \theta_k = \theta_j + \log(1 + e^{\theta_k'}).
\end{equation}

Although it goes beyond our discussion here, fitting these types of non-linear parameterizations (and especially transformations that deal with complex combinations of parameters) is largely the goal of \emph{deep learning}.

We will see examples of non-linear models in later chapters (e.g.~\Cref{sec:rnn},\Cref{sec:modelbased}) though for the moment we briefly discuss

%\todo{double check code}
%\todo{only useful if there's comparible code elsewhere}
\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
y = tf.constant(y, shape=[len(y),1]) # Convert to column vector
M = len(X[0]) # Feature dimensionality

# Regularized MSE, described using tensorflow matrix operations
def MSE(X, y, theta, lambda):
  return tf.reduce_mean((tf.matmul(X, theta) - y)**2) +
         lambda * tf.reduce_sum(theta**2)

# Initialize theta
theta = tf.Variable(tf.constant([0.0]*M, shape=[M,1]))

# Adam Optimizer
optimizer = tf.train.AdamOptimizer(0.01)
objective = optimizer.minimize(MSE(X,y,theta,1.0))
\end{lstlisting}
\end{minipage}
%\todo{Don't think AdamOptimizer uses this syntax anymore}
%\todo{Don't think "session" exists either}
%\todo{Replace with tf.nn.l2_loss (and elsewhere)}
%\todo{Has regularizer but haven't discussed regularization yet}

\subsection{Case Study: Image Popularity on Reddit}

\cite{lakkaraju2013s} used regression algorithms to estimate the success of content (e.g.~number of upvotes) on \emph{reddit}.

%\todo{connecting sentence}

Presumably, one of the biggest predictors of success is the quality of the content itself. Predicting whether submitted content 
%(e.g.~an image)
is of high quality (e.g.~whether an image is funny or aesthetically attractive) is presumably incredibly challenging.
To control for this high-variance factor of content quality, 
%the primary source of data studied in 
\cite{lakkaraju2013s} study \emph{resubmissions}, 
%on reddit, 
i.e.,~content (images) that has been submitted multiple times. 
%
%Studying resubmissions allows the study to control for the influence of the content itself
This way, if one submission is more successful than another (of the same image), the difference in success cannot be attributed to the content itself, and must arise due to other factors such as the title of the submission or the community it was submitted to.

%Community effects:
Having controlled for the effect of the content itself, the goal is then to distinguish between features that capture the specific dynamics of reddit itself, versus those that arise due to the choice of title (i.e.,~how the content is `marketed'). Various features are extracted that model the reddit community dynamics, such as the following:
%\todone{could connect a bit better}
\begin{itemize}
    \item One of the largest predictors of successful content is simply whether it has been submitted before (as we saw in \Cref{fig:redditResub}, which is based on the same dataset); this is captured via an exponentially decaying function.
    \item However, the above effect might be mitigated if enough time has passed between resubmissions (by when the original submission is forgotten, or the community has enough new users); this is captured using a feature based on the \emph{inverse} of the time delta between submissions.
    \item Resubmissions might still be successful if they are resubmitted to largely non-overlapping communities (subreddits).
    \item Submission success may correlate with the time of day. For example, 
    %success may correlate with overall traffic on the site, 
    submissions may be most successful during the highest-traffic times of day,
    or alternately they may be more successful if submitted when there is less competition.
\end{itemize}
%\todo{check style on all items: do they finish with a . ?}

Whereas community effects are somewhat reddit-specific, a question of broader interest is to determine the effect 
%\todo{effect vs affect, check everywhere}
that a particular choice of \emph{title} has on the success of a submission. Understanding the characteristics of a successful title can have broader implications when marketing content (e.g.~an advertising campaign) to a new market (for example).
%\todo{repetitive compared to previous paragraph}

%Title effects:
Features are extracted to capture the dynamics of a submission's title, including:
\begin{itemize}
    \item Titles should differ from those previously used in submissions of the same content.
    \item Titles should align with the expectations of the community the content is submitted to. Interestingly, \cite{lakkaraju2013s} find that there is a `sweet spot,' in the sense that titles should roughly follow the linguistic style of previous successful submissions in the same community, but should not be \emph{too} similar, to the point that they are not novel compared to previous submissions.
    We'll discuss text similarity measures more in \Cref{chap:text}.
    \item Successful titles might have other features, in terms of length, sentiment, linguistic style, etc.
\end{itemize}

Ultimately, all of the above features are combined into a regression model that estimates the score (number of upvotes minus number of downvotes) that a particular submission will receive.

Due to the way that features are combined, the model is not linear in the parameters, so optimization proceeds by gradient descent (as in \Cref{sec:gradient}). The method is evaluated in terms of the $R^2$ coefficient, with experiments revealing that community and textual features both play a key role in prediction. Finally, it is shown that the method can be used `in the wild' to predict the success of actual reddit submissions.

\section{The Learning Pipeline}

By now we have covered many of the individual components that go into building a predictive model: model fitting (\Cref{sec:?}), feature engineering (\Cref{sec:?}), and evaluation (\Cref{sec:?}). Bringing these components together still requires filling in some additional details. How can we know whether our model will work well when deployed (i.e.,~on \emph{new} data), and what steps can be taken to ensure this? How can we decide between various alternatives in terms of feature design, and meaningfully compare those alternatives against each other? 
%
Collectively these steps are part of the \emph{pipeline} of machine learning.
%\todo{fluff paragraph}

\subsection{Generalization and Overfitting}

So far, when discussing model evaluation in \Cref{sec:evaluateRegression}, we've considered training a model to predict labels $y$ from a dataset $X$; we've then evaluated the model by comparing the predictions $f(x_i)$ to the labels $y_i$. Critically, we're using the \emph{same} data to train the model as we're using to evaluate it.

The risk in doing so is that our model may not \emph{generalize} well to new data. For example, when fitting a model relating review length to ratings (as in \Cref{fig:ratingLengthBestFit?}), increasing the degree of the polynomial would continue to lower the errors of the predictor; alternately, we could have modeled review length using a one-hot encoding (so that there was a different predicted value for every length). In spite of their low MSEs (or high $R^2$ coefficients), these models would not seem to meaningfully capture the \emph{trends} in the data; rather, they are effectively `memorizing' meaningless details in the observed data.

To consider an extreme case, imagine fitting a vector $y$ using only \emph{random} features. The code below fits a vector of 50 observations using 1, 10, 25, and 50 random features, and then prints the $R^2$ coefficient of each model:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
y = numpy.random.rand(50)
mod = linear_model.LinearRegression()
for n in [1,10,25,50]:
    X = numpy.random.rand(50,n)
    mod.fit(X,y)
    print(mod.score(X,y))
\end{lstlisting}
\end{minipage}
Here, the $R^2$ coefficients take values of 
% 0.07321723526307
% 0.24827931383412216
% 0.3478616991354071
% 1.0
0.07, 0.25, 0.35, and 1.0---once we include 50 random features, we can fit the data perfectly. Of course, given that our features were \emph{random}, this `fit' is not meaningful, and the model has merely discovered random correlations between the observed data and labels.

The above points to two issues that need to be addressed when training a model:
\begin{enumerate}
    \item We should not evaluate a model on the same data that was used to train it. Rather we should use a held-out dataset (i.e.,~a \emph{test} set).
    \item Features that improve performance on the training data will not necessarily improve performance on the held-out data. 
\end{enumerate}
%\todo{Do I ever use enumerates or always itemize?}
Evaluating a model on held-out data gives us a sense of how well we can expect that model to work `in the wild.' Thus we are measuring how well our model will \emph{generalize} to new data.

\paragraph{Overfitting}
%\todo{Don't like the paragraph name since it matches the subsection name}

Fundamentally, if our model works well on training data but not on held-out data, it must mean that certain characteristics of the training data 
are somehow not representative of the held-out data. This could occur for various reasons. 
One possibility is that our held-out data is drawn from a different distribution from the training data.
For instance, if we had held-out sales data from the most recent month, and trained on data from the previous eleven, it is possible that the most recent month of observations follow a different trend, or occurs during a different season, etc.
%
In principle, one might address the above simply by ensuring that the training and test sets
%\todo{have I defined test set?}
are (non-overlapping) \emph{random samples} of the data, such that both the training and test data will be drawn from the same distribution.\footnote{Though if our goal is to \emph{forecast} next month's sales, using the most recent data as our held out sample may be the most appropriate decision.}

%Addressing the second point above, if a model works well on the training data but not on held-out data, 
Even if the training and held out data are independently drawn samples from the same distribution, we may still observe significantly degraded performance on our held-out data. In such cases we are said to be 
\emph{overfitting}.

A trivial demonstration of overfitting is shown in \Cref{fig:overfit}. Here we show a dataset that follows a line, subject to some random perturbation. While a high-degree polynomial can fit the data very closely, we would not expect this complex function to generalize well to new data. This again demonstrates the principle of
Occam's Razor, as we saw in \Cref{sec:regularization}, where we would expect the simpler model to better fit new data. We are said to have \emph{overfit} when we fit a model that is highly accurate on the training data, but that does not generalize well.
%\todo{too repetitive, need to read section together}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/overfit.pdf}
\caption{Overfitting demonstration. A high-degree polynomial fits the observed data accurately, but is unlikely to generalize well. \label{fig:overfit}}
\end{SCfigure}

Note that we \emph{expect} any model to perform somewhat worse when applied to new data compared to its training performance: in fact this is one of our `theorems' about model performance that we present in \Cref{sec:?}. Rather our goal when tuning the parameter $\lambda$ (\Cref{eq:accPlusReg}) using our validation set is to minimize this gap, typically by sacrificing training accuracy in order to improve generalization performance.

%\todo{Feel like text is repeated, and maybe even example. Double check}

\paragraph{Underfitting}
Just as we overfit by fitting a model whose good performance on a training set does not generalize to a held-out set, we \emph{underfit} when our model is insufficiently complex to capture the underlying dynamics in a dataset. Again, this can occur for a variety of reasons. If we select too simple a model, e.g.~a linear function to capture the data in \Cref{fig:quadCubicExample} (which doesn't seem to follow a linear trend), no choice of parameters will lead to good training \emph{or} held-out performance. Or, as above, if we tune $\lambda$ (\Cref{eq:accPlusReg}) poorly ($\lambda$ too large in the case of underfitting), we will sacrifice training performance unnecessarily by choosing too simple a model.
%\todo{style: held-out vs testing, this section}
%\todo{not the cleanest}

%\todo{finish} Whole section

\begin{figure}
    \centering
\begin{tikzpicture}
\draw (0,1.6) rectangle (10,1.1) node[pos=.5] {Dataset $\lbrace X; y \rbrace$};
\draw (0,1) rectangle (5,0.5) node[pos=.5] (train) {Training Set};
\node [below=of train,yshift=3em] {Estimate $\theta$};
\draw (5.125,1) rectangle (7.375,0.5) node[pos=.5] (valid) {Validation Set};
\node [below=of valid,yshift=3em] {Choose $\lambda$};
\draw (10,1) rectangle (7.5,0.5) node[pos=.5] (test) {Test Set};
\node [below=of test,yshift=3em] {Evaluation only};

\end{tikzpicture}
    \caption{Basic roles of training, validation, and test sets.}
    \label{fig:trainValidTest}
\end{figure}




\subsection{Model Complexity and Regularization}
\label{sec:regularization}

So far we have talked vaguely about what it means for a model to be `too complex' and suggested that we should choose a model that is complex enough to fit the data, but simple enough not to overfit.
%\todo{have we actually discussed those things?}
This idea is often referred to as \emph{Occam's Razor}, a philosophical principle which states that among several alternate hypotheses that explain some phenomenon, one should favor the simplest.

However for these notions to be useful we must be precise about what it means for a model to be `complex.' We would like to define complexity in terms of the parameters $\theta$, such that given a fixed set of features and labels, we could select the `simplest' $\theta$ that adequately explains (or models) the data.

We'll discuss two candidate notions of `simplicity' as follows:
\begin{itemize}
    \item A simple model is one in which all terms are about equally important, i.e.,~one in which all terms $\theta_k$ are about equally large.
    \item A simple model is one that includes only a few terms, i.e.,~in which only a few values $\theta_k$ are non-zero.
\end{itemize}
Below we'll argue that these two notions of `complexity' are captured by the following expressions:
\begin{eqnarray}
    \Omega_2(\theta) & = \| \theta \|_2^2 & = \sum_k \theta^2_k\\
    \Omega_1(\theta) & = \| \theta \|_1 & = \sum_k | \theta_k |,
\end{eqnarray}
i.e.,~the sum of squares and sum of absolute values.

%Explanation:
%https://stats.stackexchange.com/questions/45643/why-l1-norm-for-sparse-models


%\todo{A little more structure, maybe paragraph headings or similar}

In order to fit a model which simultaneously explains the data but is not overly complex (corresponding to our goal above), we write down a new objective that combines our original accuracy objective from \Cref{eq:?} with one of the complexity expressions above:
\begin{equation}
    \underbrace{\frac{1}{|y|}\sum_{i = 1}^{|y|} (x_i \cdot \theta - y_i)^2}_{\text{accuracy}} + \lambda \underbrace{\sum_k \theta^2}_{\mathclap{\text{model complexity}}}
    \label{eq:accPlusReg}
\end{equation}

Note that generally speaking the term $\theta_0$, i.e.,~the \emph{offset} term, should not be included in the regularizer, i.e.,~our regularizer should be $\sum_{k=1}^K \theta_k^2$ or $\sum_{k=1}^K |\theta_k|$. That is, our underlying assumption that few parameters are non-zero, or that parameters are small, should not apply to the offset term. If we were to include the offset term when regularizing, we would generally select a model which made systematically smaller (in magnitude) predictions.
%\todo{cite exercise}

\paragraph{Validation Sets}
We now require a protocol for choosing the best value of the trade-off parameter $\lambda$ in \Cref{eq:accPlusReg}. If we were to select $\lambda$ based on the accuracy on the training set, we would always select $\lambda = 0$, and 
Ideally, we want to choose 
the value of 
$\lambda$ 
%in \Cref{eq:accPlusReg} 
that will result in the best performance on the held-out test set. However recall
%\todo{check that we're recalling from somewhere}
that we cannot use the test set to compare alternative models: the test set is supposed to represent true held-out performance, and strictly speaking test performance should only be examined \emph{after} we have selected our best model.
%\done{I think}

As such, we need a third partition of our data which can be used to select among alternative models. This \emph{validation set} 
in some sense mimics the test set, in that it is not used to fit the model, but is used to give us an \emph{estimate} of what we expect the test performance to be under certain modeling conditions.

As such, a typical pipeline will consist of three partitions of our data, whose roles are summarized as follows:
%\done{need to move probably}
\begin{itemize}
  \item The \textbf{training set} is used to optimize the parameters of a specific model. In the type of models we fit in this book, this usually refers to the parameters that can be fit via gradient ascent/descent (i.e.,~$\theta$ in this chapter).
  \item The \textbf{validation set} 
  %\todo{capitalization?}
  is used to select among model alternatives. `Model alternatives' may simply mean different values of $\lambda$ in \Cref{eq:accPlusReg}, but could also mean different feature representation strategies, etc. We discuss a few alternative uses below. Typically we select the model with the highest accuracy/lowest error on the validation set.
  \item The \textbf{test set} is used to evaluate held-out model performance; ideally it should not be used to make any modeling decisions, but should only be used to report performance.
\end{itemize}

The relationship between these three sets is shown in \Cref{fig:trainValidTest}, and some overall guidelines for building a validation set are shown in \Cref{box:validationGuidelines}.

%\todo{structure in this section}

\paragraph{Why does the $\ell_1$ norm induce sparsity?}
%\todo{probably put somewhere else}

%\todo{funny paragraph name}
%\todo{connecting text / paragraph. Put this paragraph in the right place}
\Cref{fig:l1l2} demonstrates (for a simple two-parameter model) why the $\ell_1$ norm induces sparsity (i.e.,~few non-zero paramaters) while the $\ell_2$ norm results in a more uniform parameter distribution. Models with equivalent $\ell_1$ norm lie along a diamond-shaped contour (\Cref{fig:l1l2}, left) whereas models with equivalent $\ell_2$ norm lie on a circle (\Cref{fig:l1l2}, right). Models with an equivalent mean squared error
%\todo{style: mean-squared, mean squared, Mean Squared, etc.}
lie along an ellipse. When balancing the error and the regularizer as in \Cref{eq:accPlusReg}, the best model will correspond to the point where the boundaries intersect. In the case of the $\ell_1$ norm these curves intersect on \emph{on one of the vertices of the diamond}; for the $\ell_2$ norm they do not. The former case corresponds to a model
%whereas for the $\ell_2$ norm 
%corresponding to a model 
with only a few non-zero parameters.
For a more rigorous explanation of this phenomenon see e.g.~\cite{friedman2001elements}.

\begin{figure}
    \begin{center}
    \includegraphics[scale=0.75]{fig/normDemo1.pdf}
    \includegraphics[scale=0.75]{fig/normDemo2.pdf}
    \end{center}
    \caption{Demonstration of the regularization effect of the $\ell_1$ (left) versus $\ell_2$ norms. Dashed lines indicate models with equivalent norms ($\ell_1$ or $\ell_2$); solid lines indicate models with equivalent mean-squared errors. The selected model in either condition is circled. \label{fig:l1l2}}
    %\todo{kinda ugly plots. Could remove boundaries and place \theta_1, \theta_2 along axes}
\end{figure}


\begin{figure}
\caption{Guidelines for building training, validation, and tests sets}
\begin{itemize}
    \item Training, validation, and test sets should (generally speaking) be non-overlapping, random samples of a dataset. Some exceptions apply, for example when fitting temporal models in \Cref{chap:temporalSequential}, we might build a test set out of the \emph{most recent} observations in order to get a sense of how well a model would work \emph{now} rather than how well it would work \emph{on average}.
    \item The size of our training set may be driven by modeling as well as practical concerns. Our training set should be large enough that we can reasonably expect to fit our model on the data (as a guideline, we might hope to have an order of magnitude more training examples than model parameters); likewise if we have a simple model with just a few parameters we need not train on millions of observations.
    \item Likewise the size of our validation and test sets should be large enough that we can be reasonably confident of our results. We briefly touch upon measuring significance in \Cref{sec:?}
\end{itemize}
\label{box:validationGuidelines} 
\end{figure} 
%\done{I think}
%\todo{make sure all figures/tables/boxes referenced}


\paragraph{`Theorems' Regarding Training, Testing, and Validation Sets}

To solidify the roles of training, validation, and test sets, below we outline some theorems guiding the relationships among these sets, as the regularization hyperparameter $\lambda$ changes.

Note that these are `theorems' in the sense that they will be true in general, but only in the limit given large enough datasets, and that our training, validation, and test sets are drawn from the same distribution (etc.). As such, these theorems should mostly be regarded as guidelines to `sanity check' the correctness of your model pipeline:
%\todo{anything more to say?}
\begin{itemize}
    \item The training error increases as $\lambda$ increases; typically for large $\lambda$ it will asymptote to some value, e.g.~a linear model might asymptote to the error of a trivial predictor (i.e.,~the variance of the label).
    \item The validation and test errors will be at least as high as the training error; intuitively, the algorithm will not work better on `unseen' data than it did on training data.
    \item When $\lambda$ is too small, a too-complex model achieves low training error, but high validation/test errors. In this case, the model is said to be \emph{overfitting}.
    \item When $\lambda$ is too large, a too-simple model has high training, validation, and test error. In this case, the model is said to be \emph{underfitting}.
    \item Generally, there should be a `sweet spot' between under- and over-fitting, which is determined using our validation set. This point (marked in \Cref{fig:validation} with an `x') corresponds to the model we expect to yield the best generalization performance on the test set.
\end{itemize}

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/validation.pdf}
    \caption{Example train, validation, and test curves, demonstrating the relationships between each type of error.}
    \label{fig:trainValidCurves}
\end{figure}
%\todo{figure referenced?}


\subsection{Guidelines for Model Pipelines}

Having introduced the conceptual details of a model pipeline, it is worth finishing with some practical advice on how to combine the various pieces and how to set the various tunable components of a model pipeline:
\begin{itemize}
    \item If you do \emph{not} see a `sweet spot' between under- and over-fitting (as in the theorems above), it could mean that you have not adequately explored the range of regularization coefficients. For example, if you observe monotonically increasing validation errors, it may mean you have not considered sufficiently small values of $\lambda$. Alternately, given (e.g.)~a simple linear model with only a few parameters, it may simply mean that your model is \emph{not capable} of overfitting to a particular dataset.
    \item Regularization parameters such as $\lambda$ do not have an absolute scale, and will vary depending on factors ranging from the model's tendency to overfit, to the specific scale of the features $x$ and labels $y$.
    %\todo{formatting of x and y}
    As a rough guideline, it is useful to consider setting $lambda$ by considering several different orders of magnitude (as we do in \Cref{sec:pipelineExample}), before honing in on a narrower range of values.
    \item When implementing iterative models (such as approaches based on gradient descent, as in \Cref{sec:?}), the validation set can be used as a condition to cease further iteration. That is, we need not train models until convergence: if we are making no further improvements on the validation set (say, for a predetermined number of iterations), there is little reason to continue optimizing our model on the training set. Ideally, the model parameters $\theta$ might be chosen from whichever iteration yields the best \emph{validation} performance.
\end{itemize}
%\todo{more?}

\paragraph{Uses of a Validation Set}
%\todo{Fold this and the above section together}

The core use of our validation set is to estimate model hyperparameters such as $\lambda$ above. Beyond regularization coefficients, `hyperparameters' more broadly refer to any tunable model components that do not get optimized during the training phase. For example, when building models from text in \Cref{chap:text}, the number of words in our dictionary (from which we build features) would be an example of a hyperparameter, and could be chosen using our validation set to select the model that will generalize the best.

Our validation set can also be used when training a model. For instance, when we fit models via gradient descent (as in \Cref{sec:gradient}), we might periodically evaluate the model's validation performance every few gradient iterations; rather than waiting until the model reaches a particular convergence threshold, we can simply terminate once performance is no longer improving on the validation set.
%\todo{Short and not useful section, could be folded into something else -- fold into above as previous todo says}

\subsection{Significance Testing}

Although not a focus of this book, it is worth briefly exploring how we can formally determine whether the performance of one model is `better' than another, in terms of a formal statistical framework. So far, we have compared models in terms of their mean-squared errors
%\todo{style, hyphenation}
though as we explored in \Cref{sec:?}, the mean squared error was chosen based on an underlying assumption that model errors are randomly distributed (under a Gaussian distribution).

\emph{Significance testing} refers to the overall process of determining whether a statistical measurement would have been likely to have occurred due to chance alone (under the assumptions of some particular model). For example, 
%if two restaurants on \emph{Yelp}
%\todo{italics}
%had average ratings of 4.5 and 4.3, but the former had two reviews while the latter had 50, which would you conclude was better? 
if an established restaurant on \emph{Yelp}
%\todo{italics}
has a rating of 4.3 stars based on 50 reviews, and a new restaurant has a rating of 4.5 stars based on four reviews, would you conclude that the new restaurant is better rated? Or would you conclude that the higher initial rating is likely to have occurred due to chance? Significance tests allow us to formalize these questions.

Formally, a \emph{$p$-value} measures the probability that a result as (or more) extreme as the one we actually observed could have occurred due to chance (under some statistical model). E.g.~if we assume that users' ratings follow a Gaussian distribution, with what probability would the ratings of two restaurants deviate by more than 0.2 stars? 
In the case of this specific measurement, this probability would depend on (a) the 
%\todo{(abc) or (123)?}
\emph{magnitude} of the difference between the two averages; (b) the \emph{size} of the two samples (e.g.~a difference of 0.2 stars might be significant of both restaurants had 50 ratings, but not if they had four); and (c) the \emph{variance} of the two samples (e.g.~if the two samples had highly concentrated ratings we might more quickly conclude that the difference was significant).\footnote{This specific probability would be measured via a $t$-test.}

When making comparisons between models, 
%we are typically interested in 
we will typically use a $p$-value to measure
whether one model has residuals ($f(x) - y$)
%\todo{notation}
that are closer to zero than another (i.e.,~we are testing whether one model's predictions are closer to the labels than the other's). To do so we are measuring the difference in \emph{variance} between two samples.

%whether the difference between the mean squared errors of two models is likely to have occurred due to chance.

%WRONG
%Without going through the derivation here, this specific $p$-value (between two random samples that are assumed to follow a Gaussian distribution) is computed by a $t$-test, and indeed is a function of the means, variances, and sizes of the two samples.

We will compute this quantity via an \emph{$F$-test}. Other tests could also be used to compare the performance of two models, such as a likelihood ratio test, each of which has different underlying assumptions.
%\todo{clunky}
Below we'll compare the performance of two models for estimating a rating, using our beer review data from \Cref{sec:?}:
\begin{eqnarray}
    \text{model 1:} \quad \text{rating} & = & \theta_0 \\
    \text{compared to model 2:} \quad \text{rating} & = & \theta_0 + \theta_1 \times \text{ABV}
    %\todo{formatting / style}
\end{eqnarray}
%\todo{remove all eqnarray}
One of the assumptions of this particular test is that one of the to models has a subset of the parameters of the other. As such we are really measuring whether the additional parameters significantly improve the model's performance (i.e.,~whether adding a term based on the ABV improves the performance of a model including only $\theta_0$).

First we generate features and labels for the two models (assuming the data has already been read, shuffled, etc.):

\begin{minipage}{\linewidth}
\begin{lstlisting}
X1 = [[1] for d in data]
X2 = [[1, d['beer/ABV']] for d in data]
y = [d['review/overall'] for d in data]
\end{lstlisting}
\end{minipage}

Next we fit the two models (on half of the data), and compute their residuals (on the other half):

\begin{minipage}{\linewidth}
\begin{lstlisting}
model1 = sklearn.linear_model.LinearRegression(fit_intercept=False)
model1.fit(X0[:250], y[:250])
residuals1 = model1.predict(X1[250:]) - y[250:]
model2 = sklearn.linear_model.LinearRegression(fit_intercept=False)
model2.fit(X2[:250], y[:250])
residuals2 = model2.predict(X2[250:]) - y[250:]
\end{lstlisting}
\end{minipage}

The actual $F$ statistic depends on the sum of squared residuals, the number of parameters in each model, and the size of the sample:

\begin{minipage}{\linewidth}
\begin{lstlisting}
rss1 = sum([r**2 for r in residuals1]) # sum of squared residuals
rss2 = sum([r**2 for r in residuals2])
k1,k2 = 1,2 # Number of parameters of each model
n = len(residuals1) # Number of samples
\end{lstlisting}
\end{minipage}

Finally we compute the $F$ statistic and estimate the associated $p$-value using a method from \texttt{scipy}:
%\todo{formatting}

\begin{minipage}{\linewidth}
\begin{lstlisting}
F = ((rss1 - rss2) / (k2 - k1)) / (rss2 / (n-k2))
scipy.stats.f.cdf(F,k2-k1,n-k2)
\end{lstlisting}
\end{minipage}

The obtained $p$-value is close to zero, indicating that the result (that ABV improves predictive performance) is statistically significant.\footnote{One can experiment with less-predictive features or a smaller test set to see how these influence the estimated $p$-value.}

Note that this is just one example of a significance test, that works for a particular situation (albeit a fairly common one); in different situations alternative tests may be required, see e.g.~\cite{wasserman2013all} for a more comprehensive presentation.

In spite of the importance of rigorously demonstrating the significance of claimed model improvements,
%is in general an important best-practice to follow, 
we will mostly avoid further discussing significance testing throughout the remainder of this book. Generally speaking, these types of tests are designed for small-sample contexts (e.g.~surveys or clinical trials, etc.);~on the types of large datasets we consider, even small differences between models will tend to yield extremely small (highly significant) $p$-values.
%as opposed to machine learning on large datasets)

\section{Classification}
\label{sec:classification}

So far, we have considered supervised learning tasks in which the output variable $y$ is a real number, i.e.,~$y \in \mathbb R$. Often, we will deal with problems with binary or categorical output variables, for example we might be interested in problems such as:
\begin{itemize}
  \item Will a user click on a product or advertisement? (binary outcome)
  \item What category of object does an image contain? (multiclass)
  \item What product is a user most likely to purchase next? (multiclass)
  \item Which of two products would a user prefer? (binary)
\end{itemize}

\subsection{Logistic Regression}
\label{sec:logisticRegression}

We will focus on a form of classifier that extends the ideas behind regression to classification problems.
%\todo{Move the figure below}

%\todo{Explain that logistic regression is an example of a linear classifier}

Logistic Regression sets up classification using a probabilistic framework; the predictions $X\theta$ that we used when building regressors are transformed into \emph{probabilities} of observing a particular label.

Ultimately, logistic regression is just one of dozens of classification schemes; we describe it here rather than other popular alternatives (such as Support Vector Machines \cite{cortes1995support}, or Random Forest Classifiers \cite{ho1995random}) mainly because logistic regression more closely matches the approaches we will develop in later chapters.
%\todo{more or less repeated text}

In short, logistic regression associates a probability of the dataset under some model, which is fully differentiable and can be optimized using gradient-based approaches (as we saw in \Cref{sec:whyMSE,sec:gradient}).
%\todo{check}
This same type of modeling approach will be used throughout this book, when building Recommender Systems in \Cref{chap:recsys}, or 
%modeling visual feature 
%or defining model fairness objectives
%in \Cref{chap:ethics}.
generating fashionable outfits in \Cref{chap:visual}.
%\todone{could say more}

\paragraph{Fitting a logistic regressor}
When fitting regular linear regressors, we wanted a model $f_\theta$ whose estimates $f_\theta(X_i)$ were as close as possible to the (real-valued) labels $y_i$. When adapting a linear regression algorithm  to classification, we might instead seek models that associate positive values of $X_i \cdot \theta$ with positive labels ($y_i = 1$), and negative values of $X_i \cdot \theta$ with negative labels ($y_i = 0$).

If we could do so, we could write down the \emph{accuracy} associated with a particular model:
%\todo{renders ugly}
\begin{equation}
\frac{1}{|y|}
    \sum_{i=1}^{|y|} \underbrace{\delta(y_i = 0) \delta(X_i\cdot \theta \leq 0)}_{\hspace{-15mm}\mathrlap{\text{label is positive and prediction is positive}}} + \overbrace{\delta(y_i = 1) \delta(X_i\cdot \theta > 0)}^{\mathllap{\text{label is negative and prediction is negative}}\hspace{-15mm}}
    \label{eq:accuracySummation}
\end{equation}
(here $\delta$ is an indicator function that returns $1$ if the argument is true, $0$ otherwise). The equation above, in spite of slightly confusing notation, is merely counting the number of times we correctly predict a positive score for positively labeled data, and a negative score for negatively labeled data.

We now simply desire from our classifier $\theta$ that it maximizes the accuracy in \Cref{eq:accuracySummation}.
Unfortunately, directly optimizing the accuracy in \Cref{eq:accuracySummation} is NP-hard (see e.g.~\cite{nguyen2013algorithms}). To get a sense for why it is difficult, consider that the function in \Cref{eq:accuracySummation} is essentially a \emph{step} function (\Cref{fig:sigmoid}, left), i.e.,~it is flat (deravitive zero) almost everywhere; it is therefore not amenable to techniques like gradient ascent.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/step.pdf}
\includegraphics[scale=0.75]{fig/sigmoid.pdf}
\caption{The sigmoid function (right), and the step function (left) that it approximates. \label{fig:sigmoid}}
\end{SCfigure}

So, to optimize the accuracy approximately, we would like a function that is \emph{similar to} \Cref{eq:accuracySummation}, but is more straightforward to optimize.

\emph{Logistic Regression} achieves this goal by converting the outputs of a linear function $X_i \cdot \theta$ to \emph{probabilities} via a smooth function. Our intuition is that large values of $X_i \cdot \theta$ should correspond to high probabilities, and small (i.e.,~large negative) values of $X_i \cdot \theta$ should correspond to small probabilities.

This goal can be achieved via the \emph{sigmoid function}:
\begin{equation}
    \sigma(x) = \frac{1}{1 + e^{-x}}.
\end{equation}
This function, depicted in \Cref{fig:sigmoid}, maps a real value to the interval $(0,1)$, and passes through 0.5 when $x=0$. Thus it can be interpreted as a probability:
\begin{equation}
p_\theta(y_i = 1|X_i) = \sigma(X_i \cdot \theta) = \frac{1}{1 + e^{-X_i\cdot \theta}}
\end{equation}
Now, as a smooth surrogate for the expression in \Cref{eq:accuracySummation}, we can instead optimize
%TODO: Rewrite the previous function so that it uses this notation rather than delta functions
\begin{align}
    \mathcal{L}_\theta(y|X) & = \prod_{y_i = 1} p_\theta(y_i = 1|X_i) \times \prod_{y_i = 0} \left(1 - p_\theta(y_i = 0|X_i)\right)\\
    & = \prod_{y_i = 1} \frac{1}{1 + e^{-X_i\cdot \theta}} \times \prod_{y_i = 0} \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_i\cdot \theta}}.
\end{align}
% Our goal is to maximize this expression for $\theta$, i.e., to find
% \begin{equation}
%     \argmax_\theta L_\theta(y|X)
% \end{equation}
The above expression is a \emph{likelihood function}, much like we saw in \Cref{eq:likelihood1}.
Intuitively, for this expression to be maximized we want positive instances ($y_i = 1$) to be associated with high probabilities, and negative instances ($y_i = 0$) with low probabilities.

Our goal is to maximize the above function, i.e.,~to find $\argmax_\theta L_\theta(y|X)$. Short of a closed form solution, our approach is as follows:
\begin{itemize}
    \item Take the logarithm $\ell_\theta(y|X)$, since $\argmax_\theta \mathcal{L}_\theta(y|X) = \argmax_\theta \log(\mathcal{L}_\theta(y|X))$.
    \item \emph{Subtract} a regularizer $\lambda \| \theta \|_2^2$ (or similar).\footnote{Note that we subtract the regularizer rather than adding it as we did in \Cref{?} since we are maximizing rather than minimizing our objective; since we still want low complexity, we seek to maximize $-\lambda \| \theta \|_2^2$.}
    \item Compute the gradient of $\ell_\theta(y|X) - \lambda \| \theta \|_2^2$, and optimize via gradient ascent.
\end{itemize}
We compute the gradient below as follows:
\begin{align}
\ell_\theta(y|X) & = \sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-X_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right) - \lambda \| \theta \|_2^2 \label{eq:logisticDerive1}\\
 & = \sum_{i} - \log(1 + e^{-X_i\cdot \theta}) + \sum_{y_i = 0} -X_i \cdot \theta - \lambda \| \theta \|_2^2 \label{eq:logisticDerive2}\\
\frac{\partial \ell}{\partial \theta_k} & = \sum_i x_{ik} \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_i \cdot \theta}} - \sum_{y_i = 0} x_{ik} - \lambda\theta_k\\
& = \sum_i x_{ik} \left( 1 - \sigma(X_i \cdot \theta) \right) - \sum_{y_i = 0} x_{ik} - \lambda \theta_k
\end{align}
(note the summation indices change between
%\todo{Why do the references below point to the wrong thing?}
\Cref{eq:logisticDerive1,eq:logisticDerive2}, since both terms in \Cref{eq:logisticDerive1} have the same denominator).
%\todone{section is done, think whether paragraph below is needed}

%The basic idea behind logistic regression is that each observation $X_i$ 
%%\todo{notation?} that has a positive label ($y_i = 1$) should be associated

\paragraph{Summary}
Our development of logistic regression above is representative of the overall approach we'll take later when developing models that estimate interactions, clicks, purchases, etc.: 
\begin{itemize}
    \item Rather than estimating an outcome directly, we associate a \emph{probability} with each outcome.
    %\item 
    Associating the outcome with probability allows us to associate discrete (e.g.~$y_i \in \lbrace 0, 1 \rbrace$) 
    %\todo{notation}
    outcomes with a continuous function ($f(x) \in (0,1)$); this is accomplished via a transformation (such as the sigmoid function) which maps a real-valued output into the correct range.
    \item The model should associate positive (1) labels with high probabilities, and negative labels (0) with low probabilities. Likewise we can associate a probability to the entire dataset by taking a product of probabilities (or a sum of log-probabilities, as in \Cref{eq:?}).
    \item Ultimately the procedures above allow us to associate the quality of a model (paramaterized by $\theta$) with a continuous function whose value we should try to maximize; we optimize the model via gradient ascent.
\end{itemize}
%\todo{consider whether this paragraph is redundant}

\subsection{Other Classification Techniques}

In this introductory chapter, we have only discussed a single classification technique: Logistic Regression. Our choice to explore this particular technique was largely a practical one: the idea of associating a probability with a particular outcome (as in \Cref{eq:?} and estimating that probability via a differentiable function (to facilitate gradient ascent) will appear repeatedly as we develop more and more complex models. Likewise, the choices of training and regularization strategies were largely guided by what will prove useful later.

However the specific techniques we've explored are only one set of approaches to build classifiers, and 

\begin{description}
  \item[Support Vector Machines] While logistic regressors optimize a probability associated with a set of observed labels, they do not explicitly minimize the number of \emph{mistakes} made by the classifier. Support Vector Machines (SVMs)
  \cite{cortes1995support} replace the sigmoid function in \Cref{eq:?} with an expression that assigns zero cost to correctly classified examples, and a positive cost to incorrectly classified examples (in proportion to the confidence of the prediction $\theta \cdot x$).
  %\todo{style}
  %
  This distinction is fairly subtle: while \emph{every} sample will influence the optimal value of $\theta$ for a logistic regressor, the solution found by an SVM is entirely determined by a few samples closest to the classification boundary, or those that are mislabeled. Conceptually it is appealing for a classifier to focus on the most `difficult' samples in this way, though note that in many cases (and notably when building recommender systems) our goal is to optimize ranking performance rather than classification accuracy (see e.g.~\Cref{sec:?}), such that giving special attention to the most ambiguous examples is not necessarily desirable.
  %\todo{Best reference? Do I already talk about this somewhere?}
  \item[Decision Trees] Decision trees classify instances based on a sequence of binary decisions, each of which deals with a specific feature. 
  %Following each binary decision, 
  Each node of the tree separates the data based on such a decision, with leaf nodes being responsible for determining an outcome.
  %\todone{not great}
  Decision trees straightforwardly facilitate learning non-linear classifiers that capture complex interactions among features, e.g.~we can straightforwardly learn that 
  %for 
  %young people, low price is associated with a positive review, whereas for older people, 
  a low price is associated with a positive review for young people, while a high price is associated with a positive review for older people: such an association is difficult for a linear classifier to learn as neither the `age' nor `price' feature is individually correlated with the outcome.
  Extensions such as \emph{random forests}
  \cite{ho1995random} (an ensemble of decision trees) remain popular forms of classification. 
  %We exclude such techniques here mostly because they have little in common with the approaches we bulid in later chapters.
  %\todo{Why is this cited elsewhere? Shouldn't describe twice}
  \item[Multilayer Perceptrons] So far, we have only described \emph{linear} classifiers, which assume a simple relationship between between features and predictions. Although we argued in \Cref{sec:?} that such limitations can be overcome by careful feature engineering, ideally we might like to learn such feature transformations automatically.
  %We briefly introduce multilayer perceptrons in \Cref{sec:?} when describing their use within deep learning-based recommendation techniques.
\end{description}
%\todone{could include other approaches}

We exclude SVMs and decision trees from the remainder of this book
%such techniques here 
mostly because they have little in common methodologically with the approaches we bulid in later chapters.
We briefly introduce multilayer perceptrons in \Cref{sec:?} when describing their use within deep learning-based recommendation techniques. As we try to reiterate throughout the book, multilayer perceptrons
%\todo{style}
and various other state-of-the-art models are simply architectural choices that offer alternate ways to optimize the same objectives that we approach through simpler models. Having introduced the overall objectives, and the fundamentals of gradient based optimization approaches, adapting them to alternate architectures is (relatively) straightforward.


\section{Evaluating Classification Models}
\label{sec:evaluatingClassifiers}

So far, when developing classifiers, we have focused on maximizing the alignment between the labels and the model's outputs. E.g.~in the case of logistic regression, we want the predicted probability $p_\theta(y_i = 1 | x_i)$ to be as close as possible to the label $y_i$. Implicitly, when doing so, we are trying to maximize the model's \emph{accuracy}:
\begin{equation}
%\todo{f_\theta|y,X is not the notation I use elsewhere
    \text{accuracy}(f_\theta|y,X) = \frac{1}{N} \sum_{i=1}^{|y|} \delta ( f_\theta(x_i) = y_i ),
\end{equation}
where $\delta$ is an indicator function,\footnote{
%\todo{put this wherever delta is first used, or just in notation page}
$\delta(x) = 1$ if the argument $x$ is true, $0$ otherwise.}
and $f_\theta(x_i)$ is the binarized output of the model (e.g.~in the case of logistic regression, $f_\theta(x_i) = \delta(x_i \cdot \theta > 0)$). Equivalently we are minimizing the \emph{error}, i.e.,
\begin{equation}
    \text{error}(f_\theta|y,X) = 1 - \text{accuracy}(f_\theta|y,X).
\end{equation}
%\todo{could drop or inline error definition}

To motivate the difficulty of properly evaluating classifiers, consider the following classification task. We saw in \Cref{fig:genderLength} that there was a slight relationship between gender and review length; now, let's see if we can develop a simple classifier that attempts to predict gender based on review length:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = [[1, len(d['review/text'])] for d in data]
y = [d['user/gender'] == 'Female' for d in data]

mod = sklearn.linear_model.LogisticRegression()
mod.fit(X,y)
predictions = mod.predict(X) # Binary vector of predictions
correct = predictions == y # Binary vector indicating which predictions were correct
print(sum(correct) / len(correct))
\end{lstlisting}
\end{minipage}
%\todo{This probably shouldn't be the first time code is used in this chapter}

Surprisingly, the classifier produced by this code is 98.5\% accurate!
%\todo{Don't know if I want exclamation marks}
This result might seem implausible, but turns out to be an issue with the error measure itself. Counting the number of negative labels in the dataset

% \noindent
% \begin{minipage}{\linewidth}
% \begin{lstlisting}
% 1 - (sum(y) / len(y))
% \end{lstlisting}
% \end{minipage}
reveals that the data is 98.5\% male (i.e.,~98.5\% negative labels). Not only does this reveal that the accuracy is unlikely to be an informative metric in this case, but it reveals that \emph{our goal of optimizing the accuracy} caused us to learn a trivial classifier---the model simply predicts zero everywhere.

The above example demonstrates the problem with na\"ively computing (or optimizing) the accuracy. Several situations where we might need more nuanced evaluation measures include:
\begin{itemize}
    \item Datasets whose labels are highly imbalanced, such as the example above.
    \item Situations where different types of errors have different associated costs. E.g.~failing to detect dangerous luggage is a more severe mistake than an erroneous positive identification.
    \item When we use classifiers for search or retrieval (as we will often do when developing recommender systems), we often care about the ability of the model to confidently identify a few positive instances (e.g.~those surfaced on a results page), and are not interested in its overall accuracy.
\end{itemize}

Below we will develop error measures designed to handle each of these scenarios.

%\subsection{Type I and Type II Errors}
\subsection{Balanced Metrics for Classification}
\label{sec:type1type2}

The basic issue with the example presented above was that we allowed one of the two labels to dominate the classifier's objective. Although in some cases we may justifiably want a classifier that focuses more on the dominant label, in the example above we would likely prefer a solution that had reasonable accuracy \emph{per class}.

To achieve this we need evaluation metrics that consider the two classes (positive and negative, or female and male in our example) separately. To do so, we consider each of the four possible outcomes in terms of our prediction and label:
\begin{alignat}{3}
  \mathit{TP} & =\text{True Positives} &&= |\lbrace i | y_i \land f_\theta(x_i) \rbrace| \\
  \mathit{TN} & =\text{True Negatives} &&= |\lbrace i | \lnot y_i \land \lnot f_\theta(x_i) \rbrace| \\
  \mathit{FP} & =\text{False Positives} &&= |\lbrace i | \lnot y_i \land f_\theta(x_i) \rbrace| \\
  \mathit{FN} & =\text{False Negatives} &&= |\lbrace i | y_i \land \lnot f_\theta(x_i) \rbrace|.
  %\todo{No understanding of alignat above, and should fix equation below too}
\end{alignat}
From these, we can define errors (or accuracies) that consider each of the two classes in isolation:\footnote{Various other terms exist for these expressions, e.g.~the terms \emph{sensitivity}, \emph{recall}, \emph{hit rate}, and \emph{true positive rate} are largely interchangeable.}
%\todo{Replace by array environment and add more &'s before the last equal, and in equation above to properly align}
\begin{eqnarray}
 \mathit{TPR} & = \text{True Positive Rate} & = \frac{|\text{true positives}|}{|\text{labeled positive}|} = \frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}\\
 \mathit{TNR} & = \text{True Negative Rate} & = \frac{|\text{true negatives}|}{|\text{labeled negative}|} = \frac{\mathit{TN}}{\mathit{TN} + \mathit{FP}}\\
 \mathit{FPR} & = \text{False Positive Rate} & = \frac{|\text{false positives}|}{|\text{labeled negative}|} = \frac{\mathit{FP}}{\mathit{FP} + \mathit{TN}}\\
 \mathit{FNR} & = \text{False Negative Rate} & = \frac{|\text{false negatives}|}{|\text{labeled positive}|} = \frac{\mathit{FN}}{\mathit{FN} + \mathit{TP}}.
 %\todo{Probably change the order of the expressions in the denominators to be more consistent}
\end{eqnarray}
Note that it is trivial to optimize any one of these criteria in isolation (e.g.~we can achieve a True Positive Rate of 1.0 simply by always predicting positive). As such, we would normally optimize a criterion which considers both positive and negative labels together. One such measure is the \emph{Balanced Error Rate}, which simply takes the average of the False Positive and False Negative rates:
\begin{equation}
    \mathrm{BER}(f_\theta | y,X) =
    \frac{1}{2}(\mathit{FPR} + \mathit{FNR}) = 1 - \frac{1}{2}(\mathit{TPR} + \mathit{TNR}).
    % \frac{1}{2} \Bigl(
    %     \underbrace{\frac{\mathit{FN}}{\mathit{TP} + \mathit{FN}}}_{\mathit{FNR}} +
    %     \underbrace{\frac{\mathit{FP}}{\mathit{TN} + \mathit{FP}}}_{\mathit{FPR}}
    %     \Bigr)
    %\todo{Don't use Bigl, just use \smash and \vphantom}
    % =
    % 1 - \frac{1}{2} \Bigl(
    %     \underbrace{\frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}}_{\mathit{TPR}} +
    %     \underbrace{\frac{\mathit{TN}}{\mathit{TN} + \mathit{FP}}}_{\mathit{TNR}}
    %     \Bigr)
    \label{eq:BER}
\end{equation}
In our motivating example, this now attributes half of the error to the `Female' (positive) class and half of the error to the `Male' (negative) class.

Note that an appealing quality of the Balanced Error Rate is that (unlike the accuracy), it can no longer be minimized via trivial solutions: always predicting `True,' or always predicting `False,' or predicting at random, will result in a BER of 0.5.

\subsection{Optimizing the Balanced Error Rate}

Having argued
%(\Cref{sec:type1type2})
that the Balanced Error Rate may be preferable to the accuracy if we wish to avoid trivial solutions, we next ask how to train a classifier to avoid producing trivial solutions in the first place.

Intuitively, the degenerate solutions we saw in 
\Cref{sec:}
%\todo{this}
arose due to an imbalance in our training data (i.e.,~a high ratio of positive or negative labels). Trivially, we might correct this by \emph{re-sampling} our training data: i.e.,~sampling either a fraction of our negative negative instance, or sampling negative instances (with replacement) until we have an equal number of positive and negative instances.

While the above is a common and reasonably effective strategy, the same goal can be achieved more directly simply by \emph{weighting} the positive and negative instances. Note that in our objective for logistic regression:
\begin{equation}
    \sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-X_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right),
\end{equation}
the two summations (over $y_i = 1$ and $y_i = 0$) essentially reward the model for correctly predicting positive instances 
%(i.e.,~True Positive Rate) 
and negative instances.
%(i.e.,~True Negative Rate). 
The issue with the above objective is that it will be imbalanced, and one of the two terms can dominate the expression in the event that positive or negative instances are over-represented in our dataset.

To address this, we can normalize the two expressions by the number of samples in the positive and negative classes:
\begin{equation}
    \frac{|y|}{2|\lbrace i | y_i = 1 \rbrace|}\sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-X_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-X_i\cdot \theta}}{1 + \frac{|y|}{2|\lbrace y | y_i = 0 \rbrace|} e^{-X_I\cdot \theta}} \right).
\end{equation}
%\todo{kinda sloppy notation}
%\todo{writing still a bit sloppy}
By doing so the left- and right-hand expressions have equal importance, such that \emph{all} positively labeled instances have the same importance as \emph{all} negative instances. In other words by normalizing the two expressions correspond to the True Positive \emph{Rate} and True Negative \emph{Rate}, as in \Cref{eq:BER}.
Note that in addition to normalizing by the number of samples, both sides are multiplied by $\frac{|y|}{2}$; this is not strictly necessary but is done by convention such that the total `weight' of all instances is still $|y|$.

The above can be accomplished in sklearn with the \texttt{class\_weight='balanced'}
%\todo{do I ever use texttt anywhere?}
option as follows:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = [[1, len(d['review/text'])] for d in data]
y = [d['user/gender'] == 'Female' for d in data]

mod = sklearn.linear_model.LogisticRegression(class_weight='balanced')
mod.fit(X,y)
\end{lstlisting}
\end{minipage}

Note that the same idea can be applied to problems including more than two categories, and that one can choose different weighting schemes, e.g.~to assign any desired relative weights to true positives versus true negatives.

\subsection{Error Metrics for Ranking}
\label{sec:errorRanking}

% Precision and recall, precision@k

Often, the goal of training a classifier is not merely to generate exhaustive sets of `true' and `false' instances. For example, if we wanted to identify relevant webpages in response to a query, or to recommend items that a user is likely to purchase, in practice it may not matter whether we can identify \emph{all} relevant webpages or products; rather, we might care more about whether we can surface \emph{some} relevant items among the first page of results returned to a user.

Note that the type of classifiers we've developed so far can straightforwardly be used for \emph{ranking}. That is, in addition to outputting a label ($y_i = x_i \cdot \theta > 0$ in the case of logistic regression), they can also output \emph{confidence scores} (i.e.,~$x_i \cdot \theta$, or equivalently $p_\theta(y_i = 1 | x_i) > 0.5$).
%\todo{check notation}
Thus, in the context of finding relevant webpages or products above, our goal might be to maximize the number of relevant items returned among the few most confident predictions. Furthermore, we might be interested in how the model's accuracy changes as a function of confidence; e.g.~even if the model's accuracy is low overall, is it accurate for the top 1\%, 5\%, or 10\% of most confident predictions?

\paragraph{Precision and Recall}
%\todo{Don't like subsubsections}

\emph{Precision} and \emph{Recall} assess the quality of a set of retrieved results in terms of two related objectives. Informally, \emph{precision} measures the rate at which those items `retrieved' by the model (i.e.,~those predicted to have a positive label by the classifier) are in fact labeled positively; \emph{recall} measures what fraction of all positively-labeled items our classifier predicted as having a positive label. For example, in a spam filtering setting (where positively-labeled items are spam e-mails), \emph{precision} would measure how often e-mails marked as spam are in fact spam, whereas \emph{recall} would measure what fraction of all spam was filtered.
%\todo{is this or something similar repeated somewhere nearby?}

Formally precision and recall are defined as follows:
\begin{eqnarray}
    \text{Precision} & = & \frac{| \lbrace \text{relevant items} \rbrace \cap \lbrace \text{retrieved items} \rbrace |}{|\lbrace \text{retrieved items} \rbrace|}\\
    \text{Recall} & = & \frac{| \lbrace \text{relevant items} \rbrace \cap \lbrace \text{retrieved items} \rbrace |}{|\lbrace \text{relevant items} \rbrace|}
    \label{eq:precRecDefn1}
\end{eqnarray}

Alternately it is straightforward to verify that these expressions can be rewritten in terms of the number of true-positives, false-positives, and false-negatives, as in \Cref{eq:?}:
\begin{eqnarray}
  \text{Precision} & = & \frac{\mathit{TP}}{\mathit{TP} + \mathit{FP}}\\
  \text{Recall} & = & \frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}
\end{eqnarray}

Finally we briefly show how these quantities can be computed for a given predictor (such as the one from \Cref{code:?}):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
predictions = mod.predict(X) # binary vector of predictions

numerator = sum([(a and b) for (a,b) in zip(predictions,labels)])
nRetrieved = sum(predictions)
nRelevant = sum(y)

precision = numerator / nRetrieved
recall = numerator / nRelevant
\end{lstlisting}
\end{minipage}
%\todo{better way to write without using zip?}
%\todo{Test code}

\paragraph{$F_\beta$ Score}

Note that neither precision nor recall are particularly meaningful if reported in isolation. For instance, it is trivial to achieve a recall of 1.0 simply by using a classifier that returns `true' for every item (in which case, all relevant documents are returned); such a classifier would of course have low precision. Likewise, a precision of close to 1.0 can often be achieved by returning `true' only for a few items about which we are extremely confident; such a classifier would have low recall.

As such, to evaluate a classifier in terms of precision and recall, we likely want a metric which considers both, or otherwise to place additional constraints on our classifier (as we see below).

The $F_\beta$ score achieves this by taking a weighted average of the two quantities:
\begin{equation}
    F_\beta = (1 + \beta^2) \cdot \frac{\mathrm{precision}\cdot \mathrm{recall}}{\beta^2\text{precision} + \text{recall}}.
    \label{eq:fbeta}
\end{equation}
In the case of $\beta=1$ (which is normally called simply the `$F$-score'), \Cref{eq:fbeta} simply computes the harmonic mean of precision and recall, which is low if either of precision or recall are low.

Otherwise, if $\beta \not= 1$, the $F_\beta$ score reflects a situation where one cares about recall over precision by a factor of $\beta$.\footnote{Without going into detail, this motivation leads to the specific formulation in \Cref{eq:fbeta} \cite{van1979information}.}

There are several situations where one might care about recall more than precision, or vice versa. For instance, considering the motivating examples from the start of this section, in a baggage-handling scenario we would likely care primarily about recall, and would be willing to sacrifice precision to achieve it; or, in a search or recommendation setting, we may be happy retrieving only a few items, so long as some are relevant (i.e.,~high precision but low recall).

\paragraph{Precision and Recall @ $K$}

%\todo{Rewrite below so it sounds more like precision @ K}
%\todo{Also, argument below is ugly and doesn't follow from previous paragraph}
%Often, we are interested in evaluating the precision and recall as a function of the model's confidence. 
%We motivated precision and recall by 
One of our motivating examples when defining precision and recall
considered cases where we may only have a fixed budget of results that can be returned to a user.
%That is, what is the model's precision and recall among the top $K$ predictions? 
In particular, we might be interested in evaluating the precision and recall when our classifier returns only its $K$ most confident predictions.
To do so, we begin by sorting the labels $y_i$ according to their associated confidence scores (i.e.,~$x_i \cdot \theta$):
\begin{equation}
%  (0.04961637143573233, True),
%  (0.04265739426913856, True),
%  (0.038081609220486146, True),
%  (0.01647932932266133, True),
%  (0.0027684382608090807, False),
%  (-0.0024453115029425265, True),
%  (-0.005299363587960293, False),
%  (-0.005508568447381734, False),
%  (-0.008051803313750272, True),
%  (-0.010969509446894954, True),
%
{
\setlength{\tabcolsep}{2.2pt}
\small
\begin{tabular}{lcccccccccccc}
confidence $X_i \cdot \theta$: & $\cdots$ & 0.49 & 0.42 & 0.38 & 0.16 & 0.02 & -0.02 & -0.05 & -0.05 & -0.08 & -0.10 & $\cdots$ \\
label $y_i$: & $\cdots$ & True & True & True & True & False & True & False & False & True & True & $\cdots$
\label{eq:sortedByConfidence}
%\todo{Why are the cdots not vertically aligned?}
\end{tabular}
}
\end{equation}
Such tuples of confidence scores and labels can be generated as follows (in this case for a logistic regressor as in \Cref{sec:evaluatingClassifiers}):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
confidences = mod.decision_function(X) # real vector of confidences

sortedByConfidence = list(zip(confidences,y))
sortedByConfidence.sort(reverse=True) # sorted as in %\Cref{eq:sortedByConfidence}%
\end{lstlisting}
\end{minipage}
Note that when evaluating our model's $K$ most confident predictions, we are no longer interested in whether the actual scores are greater or less than zero (i.e.,~whether the classifier would output `true' or `false'): we are only interested in the \emph{labels} among the top-K predictions.

The \emph{Precision @ $K$} and \emph{Recall @ $K$} now simply measure the precision and recall for a classifier which returns only the $K$ most confident predictions. That is, Precision @ $K$ measures what fraction of the top-$K$ predictions are actually labeled `true'; Recall @ $K$ measures the fraction of all relevant documents that are returned among the top $K$. The main difference to note (compared to the definition in \Cref{eq:precRecDefn1}) is that the number of `retrieved' documents is always $K$; that is, the `retrieved' documents are always the $K$ most confident, whether or not the classifier actually predicts a positive label (i.e.,~$x_i \cdot \theta > 0$).

%\todo{Add code for precision and recall @ k}

Unlike precision and recall,
%\todo{consistent capitalization}
precision@$K$
%\todo{Should write it like this everywhere?}
and recall@$K$ can be reported in isolation as they cannot be optimized by trivial solutions. Precision@10, for example, is an effective measure of a classifier's ability to return reasonable results among a page of 10 retrieved items. 

\paragraph{ROC and Precision/Recall Curves}

Another holisitc measure of a classifier's performance is to report the \emph{relationship} between precision and recall, or true between and false positives.

For example, the relationship between the number of True Positives and False Positives is known as the \emph{Receiver Operating Characteristic} (ROC). It is so named because of its use in evaluating the performance of radar receiver operators: as an operator's threshold for detection decreases, both their true positive and false positive rates (TPR and FPR) will simultaneously increase; thus we might evaluate a classifier by evaluating the TPR and FPR as we change the classifier's detection threshold.
%\todo{Double check argument}

The \emph{precision recall curve} is developed following a similar line of reasoning: as we lower a classifier's detection threshold, the precision will decrease while the recall increases; thus we might evaluate a classifier by examining the relationship between precision and recall as the threshold changes.

To generate these curves, we sort the predictions of our classifier by confidence (much as we did in \Cref{eq:sortedByConfidence}), which corresponds to gradually considering lower thresholds; at each step, we compute the precision and recall (i.e.,~we compute the precision and recall @ $k$ for each value of $k$). Together these values form the precision recall curve:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
for i in range(1,len(sortedByConfidence)+1):
    retrievedLabels = [x[1] for x in sortedByConfidence[:i]]
    precision = sum(retrievedLabels) / len(retrievedLabels)
    recall = sum(retrievedLabels) / sum(y)
    xPlot.append(recall)
    yPlot.append(precision)
\end{lstlisting}
\end{minipage}

Plotting these $x$ and $y$ coordinates results in the plot in \Cref{fig:PR_ROC} (right); the ROC curve can be generated similarly. 


% AUC? Hit rate? Or leave until recommender systems?
% Left until recommender systems

We revisit evaluation techniques based on ranking in \Cref{sec:?}, when we explore evaluation strategies for recommender systems.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/ROC.pdf}
    \includegraphics[scale=0.75]{fig/PRcurve.pdf}
    \caption{Examples of Receiver-Operating Characteristic (left) and Precision Recall (right) curves.}
    %\todo{Why do the plots look funny? Seem to have big fonts}
    \label{fig:PR_ROC}
\end{figure}

% 

\section{Implementing the Learning Pipeline}

Below we briefly show how to practically apply the process from \Cref{sec:pipeline?} to select a model based on training, validation, and test samples.

The actual features for this task are based on a sentiment analysis from \Cref{chap:text}, in which we predict ratings based on words in a review; for the sake of demonstrating a model pipeline it is useful to consider a problem with high-dimensional features, such that the model is prone to overfitting if not carefully regularized (in this case, we consider 1,000-dimensional features on a dataset with only 5,000 samples).

First, we randomly shuffle the dataset and split it into non-overlapping training, validation, and test portions:
%\todo{`portions' is a funny word}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
random.shuffle(data)
X = [feature(d) for d in data]
y = [d['review/overall'] for d in data]
Ntrain,Nvalid,Ntest = 4000,500,500
Xtrain,ytrain = X[:Ntrain],y[:Ntrain]
Xvalid,yvalid = X[Ntrain:Ntrain+Nvalid],y[Ntrain:Ntrain+Nvalid]
Xtest,ytest = X[Ntrain+Nvalid:],y[Ntrain+Nvalid:]
\end{lstlisting}
\end{minipage}

Next, we consider regularization coefficients $\lambda$ ranging between $\lambda = 10^{-3}$ and $\lambda = 10^{4}$. For each value, we train a model on training set and evaluate its accuracy on the validation set; during each step, we keep track of the best-performing model in terms of validation accuracy:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
bestModel = None
bestVal = None
for l in [0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]:
    model = sklearn.linear_model.Ridge(l)
    model.fit(Xtrain, ytrain)
    predictValid = model.predict(Xvalid)
    MSEvalid = sum((yvalid - predictValid)**2)/len(yvalid)
    print("l = " + str(l) + ", validation MSE = " + str(MSEvalid))
    if bestVal == None or MSEvalid < bestVal:
        bestVal = MSEvalid
        bestModel = model
\end{lstlisting}
\end{minipage}

Finally, we evaluate the best-performing model (in terms of validation performance) on the test set. Note that this is the first and only time we use the test set:
%\todo{repetitive text}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
predictTest = bestModel.predict(Xtest)
MSEtest = sum((ytest - predictTest)**2)/len(ytest)
\end{lstlisting}
\end{minipage}

\Cref{fig:validationInPractice} shows the training, validation, and test performance found during the above steps; note the similarity to the hypothetical curves in \Cref{fig:trainValidCurves?}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/validationInPractice.pdf}
\caption{Training, validation, and test error on a real pipeline. \label{fig:validationInPractice}}
\end{SCfigure}
%\todo{Figure scale is not perfect, could redo}

\section{Exercises}
%\todo{Typesets ugly, with problem "section" followed by problems heading. Figure out how to avoid or add some text before the actual problems}
\label{sec:basicsExercises}
%\todo{make label apply to the right section after above change}

\begin{exercises}
 \item 
 %A few examples of regression problems and computing MSE to get started. Copy from class.
 \item Using the \emph{GoodReads} data (as in \Cref{sec?}), train a simple predictor that estimates ratings from review length, i.e.,
 $$\text{star rating} = \theta_0 + \theta_1 \times (\text{review length in characters}).$$
 Compute the values $\theta_0$ and $\theta_1$, and the Mean Squared Error of your predictor.
 \item Re-train your predictor so as to include a second feature based on the number of comments, i.e.,
 $$\text{star rating} = \theta_0 + \theta_1 \times(\text{length}) + \theta_2 \times(\text{number of comments})$$
 Compute the coefficients and MSE of the new model. Briefly explain why the coefficient $\theta_1$ in this model is different from the one from Exercise \ref{?}.
 \item More easy questions
 %\todo{finish}
 \item -- Some question about generating precision / recall, and ROC curves
 \item Show that $\theta_0 = \bar{y}$ is the best possible solution for a trivial predictor (i.e.,~$y = \theta_0$) in terms of the Mean Squared Error (hint: write down the MSE of this trivial predictor and take its derivative).
 \label{ex:MSEtrivial}
 \item Repeat \Cref{eq:MSEtrivial}, but this time show that the best trivial predictor in terms of the Mean \emph{Absolute} Error (\Cref{eq:MAE}) is given by taking the \emph{median} value of $y$.
 \item In \Cref{eq:MSEMLEderiv} we motivated the choice of the MSE by explaining its relationship to a Gaussian error model. Likewise, show that minimizing the MAE is equivalent to maximizing the likelihood if errors follow a Laplace distribution (the Laplace distribution has probability density function $\frac{1}{2b} \exp\left(-\frac{|x - \mu|}{b}\right)$).
 \item (Hard) In \Cref{eq:pseudoInv} we saw how to compute a line of best fit via the pseudoinverse, $\theta = (X^TX)^{-1}X^Ty$; show that the parameters that minimize the Mean Squared Error are found by taking the pseudoinverse, i.e.,~that $\argmin_\theta \frac{1}{|y|}\sum_{i=1}^{|y|} (x_i\cdot \theta - y_i)^2 = (X^TX)^{-1}X^Ty$ (that is, find the stationary point where $\frac{\partial \text{MSE}}{\partial \theta} = 0$).
 \label{ex:hardPseudoInv}
 \item (Hard) When minimizing the Mean Squared Error with a linear model as in \Cref{ex:hardPseudoInv}, show that the residuals $r_i = (y_i - x_i \cdot \theta)$ have average $\bar{r} = 0$.
 \item 
 %Show that solving a classification problem by treating it as regression on 0/1 variables isn't effective, and reason about why
 Na\"ively, to build a classifier we might simply train a \emph{regressor} by treating labels as numerical quantities (e.g.~predicting -1/+1). %\todo{Finish, come up with an example}
 Perform a simple experiment to demonstrate that this na\"ive model does not work as well as logistic
 regression. That is, select a dataset (such as the one you used in Exercise \ref{?}), a few features, a label to predict, and an appropriate classifier evaluation metric, to show that the na\"ive classifier is outperformed by logistic regression.
 %\todo{Not written well}
\end{exercises}

%-- Converting real-world datasets to 

%-- Show relationship between pseudoinverse and derivative-based solution



% -- Show that MSE residuals have average zero

% -- Show that Laplace MAE is MLE for Laplace distribution

% -- Exercise about MSE = mean and MAE = Median



%-- Some question about generating precision / recall, and ROC curves

-- Compare a model that includes $\theta_0$ in the regularizer to one that doesn't

\stepcounter{projectCounter}
\subsection{Project \theprojectCounter: Taxicab Tip Prediction}
\label{proj:1}

Throughout the chapter, we've seen various strategies for dealing with features of different types. For our first project, let's look into building a prediction pipeline to estimate tip amounts from taxicab trips. For this project you might make use of publicly-available data such as 
the \emph{NYC Taxi and Limousine Commission Trip Record Data}.\footnote{\url{https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page}}
%\cite{?}.
%https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page
%\todo{more words}
\begin{enumerate}
    \item First, conduct \emph{exploratory analysis} of the data. Just as we have done throughout the chapter, plot the relationship between the output variable (tip amount), and various features that you think might be related to this outcome.
    \item Based on the above analysis, consider what features might be useful for prediction. Consider, for example, features associated with the time of the trip, the start and end location, and the duration/distance of the trip.
    \item How should the above features be represented or transformed? For example, how can the timestamp be represented to capture variation at the level of time of day, day of week, or even the time of year (\Cref{sec:timeFeatures}); how might you represent the start and end locations? Are there any useful derived features that are useful for prediction, e.g.~$\text{speed} = \text{distance}/\text{duration}$?
    \item Is it useful to transform the \emph{output} variable (\Cref{sec:transformOutput})? For example, rather than predicting the tip \emph{amount}, it may make more sense to predict the tip \emph{percentage}.
    \item Are there any alternatives to formulating the task as a regression problem? For example, you could cast the problem as \emph{classification} by estimating whether a tip will be above or below the median; this may be less sensitive to outliers. Discuss the advantages and disadvantages of various formulations as well as what evaluation metrics you might use.
\end{enumerate}

% Exercise:
% At the beginning of \Cref{sec:classification} 

\chapter{Recommender Systems}
\label{chap:recsys}

In \Cref{chap:basics}, when revising regression and classification, our only means of providing \emph{personalized} predictions was to extract features associated with user characteristics (e.g.~age, location, gender).
%\todo{Change this in case I break it into two chapters}
%While effective in certain scenarios, 
The success of such models largely depends on our ability to extract features that adequately explain the variation in the labels we are trying to predict. While effective in a number regression or classification scenarios, when modeling interactions in recommendation scenarios, it is less clear what features are predictive of users' actions, and less likely that those features could be collected in the first place. Consider for example:
\begin{itemize}
    \item What features would be useful to predict what movies a user would be likely to watch? `Obvious' features such as user demographics may explain only a small fraction of the variation in interactions and preferences.
    \item How would you identify the types of features that would be useful for an obscure or unusual domain? For example what features would you collect to recommend baby toys, toaster ovens, or temporary tattoos?
    %\todo{good example?}
    \item Are such features likely to be available? In practice, we will often know \emph{nothing} about a user, other than their interaction history.
    \item How can we use machine learning to make predictions when \emph{no} features are available?
\end{itemize}

%\emph{Recommender systems} are arguably the most basic and fundamental form of personalized machine learning.

\emph{Recommender systems} are a fundamental tool to try and make predictions in such
%types of 
scenarios.
At their core, recommender systems are concerned with understanding \emph{interactions} between users and items. That is, we are concerned with fitting models that take a user $u$ and item $i$ as input in order to estimate an interaction label $y$ (such as a purchase, click, or rating):
\begin{equation}
    f(u,i) \rightarrow y.
\end{equation}
%\todo{notation}
Superficially, solving such a prediction task seems no different than the regression or classification scenarios we explored in \Cref{chap:basics}: na\"ively we might imagine collecting some appropriate user or item features and applying the techniques we've already developed.
%\todo{wordy, whole paragraph}
However as we study below (\Cref{sec:?}),
%
%several differences between this 
certain characteristics of this scenario render traditional regression and classification approaches ineffective, and demand that we explore 
%design
new approaches specifically designed to capture the dynamics of interaction data. Specifically:
%\todone{combine with above paragraph}
\begin{itemize}
    \item 
    Most of the techniques we'll develop in this chapter discard features altogether, and make predictions purely on the basis of historical interactions. This owes partly to the difficulty of collecting useful features, and also to the complex semantics that underlie people's preferences and behavior.
    \item
    As such, rather than having parameters associated with features as we did in \Cref{chap:basics}, the models we'll develop here have parameters associated with individual \emph{users} (or items). This shall be our introduction to the idea of \emph{model-based} personalization, as we discussed in \Cref{sec:contextVsPerson}.
    \item
    To model users (and items) we'll introduce the concept of \emph{latent spaces} in \Cref{sec:modelbased}. %Essentially we'll develop models 
    %The models we develop are based on the principle of dimensionality reduction, 
    whereby we automatically discover hidden dimensions that explain the variation in people's opinions---without necessarily knowing exactly what the dimensions correspond to.
    %\todone{not great}
\end{itemize}

Ultimately, the models we develop in this chapter are quite different from those in previous chapters, eschewing explicit features in favor of techniques more closely related to dimensionality reduction and pattern mining. 

Roughly speaking, recommender systems operate by finding common patterns and relationships among users and items,
so that recommendations for a user can be harvested from others who have similar interactions patterns.
We first explore such approaches based on simple set-based similarity heuristics in \Cref{sec:?}, before exploring machine-learning based approaches in \Cref{sec:?}.
%\todo{not great}

Our discussion of recommender systems will form the basis of many of the models we develop throughout the remainder of this book. Although in this chapter we'll build predictive models based purely on interaction histories, later we'll show how similar models can be extended by incorporating features (\Cref{chap:contentStructure}) and temporal information (\Cref{chap:temporalSequential}). Later, as we further develop personalized models of text (\Cref{chap:text}) and images (\Cref{chap:visual}) this same notion of modeling users via latent spaces will appear repeatedly.

\section{Basic Setup and Problem Definition}
\label{sec:setup}

The typical modality of the data we are trying to model might consist of sequences of historical interactions between users and items, for example, we might have a collection of movie ratings such as:
\begin{equation}
    \begin{tabular}{llrr}
      (Julian, & \emph{The Godfather}, & 4, & Jan 4 2019)\\
      (Julian, & \emph{Pulp Fiction}, & 3, & Jan 6 2019)\\
      (Laura, & \emph{Seven Samurai}, & 5, & Jan 8 2019)\\
      (Laura, & \emph{The Godfather}, & 4, & Jan 11 2019)\\
      \multicolumn{4}{c}{$\vdots$}
    \end{tabular}
\end{equation}
which might further be anonymized in terms of user IDs, item IDs, and sequential timestamps:\footnote{The timestamp shown here is known as the \emph{unix time}, representing the number of seconds since January 1970 (in UTC); such a representation is often useful as it allows straightforward comparison between timestamps.}
\begin{equation}
    \begin{tabular}{llrr}
      (264, & 547, & 4, & 1546588800)\\
      (264, & 82, & 3, & 1546761600)\\
      (3473, & 231, & 5, & 1546934400)\\
      (3473, & 547, & 3, & 1547193600)\\
      \multicolumn{4}{c}{$\vdots$}
    \end{tabular}
\end{equation}
Such a data format is ubiquitous across many popular recommendation datasets and tasks, including (for example) the popular \emph{Netflix Prize} dataset (\Cref{sec:caseStudyNetflixTemporal}).
%\todo{right ref?}
Such data may include `side information,' such as reviews, demographic information about the users, or metadata about the movies, but often it may not. In fact, in the simplest form it may not even include ratings or timestamps.

Thus in essence the data we are trying to work with simply describes \emph{interactions} among users and content. Such interactions could describe clicks, purchases, ratings, likes, (etc.). In other settings the interactions could describe social connections among users, or even `interactions' among compatible clothing items (\Cref{sec:caseStudyClothing}).

Given interaction data such as that above, we would now like to ask questions such as:
\begin{itemize}
    \item How will Laura rate Pulp Fiction?
    \item Given that Laura liked the The Godfather, what other movies will she like?
    \item What movie is Laura likely to rate next?
\end{itemize}
Answering these questions seems difficult, as we seemingly know very little about the users and items involved. However we do know, for instance, that both Laura and Julian recently watched The Godfather, and gave it similar ratings; from this we could begin to reason that may exhibit similar preferences with regard to other movies also.

Reasoning about these types of questions, and modeling these types of interaction data, are the main goals of recommender systems.

\paragraph{How is Recommendation Different from Regression or Classification?}

In \Cref{chap:basics}, we saw several techniques that seem like they could already be used to predict outcomes like ratings and purchases. For example, predicting a rating (e.g.~of a movie) seems like a traditional regression task, and we can imagine various user and movie features that might be associated with ratings. As such, na\"ively we might try to extract user and movie features and fit a linear model of the form
\begin{equation}
    \text{rating}(\text{user},\text{movie}) = \langle \underbrace{\phi(\text{user}, \text{movie})}_{\mathclap{\text{user and movie features}}}, \theta \rangle.
\end{equation}
User features might include attributes like the user's age, gender, location, or other demographic features that might be associated with rating patterns; movie features could capture the length, MPAA rating, budget, or presence of certain actors (etc.). Assuming user and movie features can be collected independently, and since the model is linear, this could be rewritten as
\begin{equation}
    \text{rating}(\text{user}, \text{movie}) = \langle \underbrace{\phi^{(u)}(\text{user})}_{\mathclap{\text{user features}}}, \overbrace{\theta^{(u)}}^{\mathclap{\text{user parameters}}} \rangle + \langle \underbrace{\phi^{(i)}(\text{movie})}_{\mathclap{\text{movie (item) features}}}, \overbrace{\theta^{(i)}}^{\mathclap{\text{movie parameters}}} \rangle.
\end{equation}
When written this way, we can see that the prediction of the rating is the sum of two \emph{independent} predictions: one for the user (say $f(u)$) and one for the item (say $f(i)$). If we were to make recommendations based on these predictions, for example by recommending whichever unseen movie a user would give the highest rating to, i.e., \begin{equation}
  \argmax_{\mathclap{i \in {\text{unseen movies}}}} f(u) + f(i),
\end{equation}
our recommendation \emph{for every user} would simply be whichever movie had the highest predicted rating $f(i)$. In other words, every user would simply be recommended movies which had features associated with high ratings.

Critically, such a model does not \emph{personalize} its recommendations to individual users. Even if the model achieved a reasonable Mean Squared Error in terms of predicting ratings, it would not be an effective recommender system.

To overcome this limitation, a model must in some fashion capture \emph{interactions} between users and items, e.g.~how compatible is a user with a particular movie. We discussed this idea a little in \Cref{chap:nonparametric}, 
%\todo{rewrite once that chapter exists}
but investigate the idea in more detail here. 

In summary, modeling \emph{interections} between users and items is the main goal of recommender systems and is the main characteristic that differentiates them from other types of machine learning (\Cref{box:interactions}).

\begin{figure}
\caption{Recommender Systems Compared to Other Types of ML}
The main distinguishing feature of recommender systems compared to other types of machine learning is their goal of explicitly modeling \emph{interactions} between users and consumed items based on historical patterns. This feature allows the models to understand which items are \emph{compatible} with which users, and thus to make different recommendations to each user in a personalized way.
\label{box:interactions}
\end{figure}

\section{Notation and Representation}

Basic notation is established in \Cref{tab:recsysNotation}. There are several ways we can represent the interaction data described above. Formally, we might simply describe the dataset as a collection of tuples $(u,i,r,t)$, or $r_{u,i,t} \in \mathbb R$ indicating that a user $u$ entered the rating $r$ for item $i$ at time $t$.

But conceptually it is easier to think about these data in terms of sets or matrices. Set representations will be useful when establishing similarity between users in terms of sets of items they have consumed (or likewise similarity between items in terms of sets of users who have consumed them); matrix representations will be useful when developing models based on the concept of matrix factorization (or dimensionality reduction).

% \begin{table}
%     \centering
%     \begin{tabular}{ll}
%         \toprule
%          $u \in U$ & users $u$ in user set $U$\\
%          $i \in I$ & items $i$ in item set $I$\\
%          $I_u$ & set of items rated by user $u$\\
%          $U_i$ & set of users who have rated item $i$\\
%         \bottomrule
%     \end{tabular}
%     \caption{Common Recommender Systems Notation}
%     \label{tab:recsysNotation}
% \end{table}

\paragraph{Activities as Sets}

For our simplest recommendation model, we can describe users in terms of the sets of items they have interacted with, e.g.~for a user $u$:
\begin{equation}
    I_u = \text{set of items consumed by $u$};
\end{equation}
likewise, we can describe items in terms of the sets of users who have interacted with them:
\begin{equation}
    U_i = \text{set of users who consumed item $i$}
\end{equation}

\paragraph{Activities as Matrices}

%\todo{Explain that sets can't handle ratings}

Alternately, we can represent a dataset of user/item interactions via matrices. 
%\todo{rewrite with both a binary matrix and a rating matrix, then can write the set notations in terms of these}
%E.g.~a matrix encoding which i:
\begin{equation}
    R = \underbrace{\left[ \begin{tabular}{ccccc}
    5 & $\cdot$ & $\cdot$ & 2 & 3\\
    $\cdot$ & 4 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 5 & 5 & 3 & $\cdot$\\
    5 & $\cdot$ & 4 & $\cdot$ & 4\\
    1 & 1 & $\cdot$ & 4 & 5\\
    \end{tabular} \right]}_{\text{items}} \left. \mathclap{\begin{tabular}{c} \\ \\ \\ \\ \\ \end{tabular}} \right\rbrace \text{users}.
    \label{eq:interactionMatrix}
\end{equation}
Each row of $R$ represents a single user, and each column represents a single item. Then a particular entry $r_{ui}$ indicates the rating a particular user $u$ gave to an item $i$. Note that the vast majority of entries in such a matrix would typically be missing (most users do not rate most items); indeed, the missing entries are exactly the quantities that we would like to predict.

Naturally, the set and matrix representations can be written in terms of each other, for example, our set representation is equivalent to:
\begin{equation}
    I_u = \lbrace{ i | R_{u,i} \neq 0 }
\end{equation}

Both our set and matrix representations of interactions possibly seem limited---neither conveys the timestamps associated with the ratings (or any other side-information), and the set-based representation doesn't even encode users' ratings. Nevertheless they are useful for reasoning about the basic principles behind recommender systems (which will become the building blocks behind more sophisticated approaches), and in fact these simple techniques are among the most widely deployed.
%\todo{Not beautiful sentenc, lacks evidence}

\section{Neighborhood-based Approaches to Recommendation}
\label{sec:neighborhood}

%\todo{Maybe call this section memory based rather than neighborhood based} 
Perhaps the simplest (and most ubiquitous) approaches to recommendation are based on some notion of `similarity' among items. That is, an item is recommended to a user because it is similar to one that they they have recently clicked, liked, or consumed.

`People who viewed X also viewed Y' (or `people who bought X also bought Y', etc.)~features (\Cref{fig:alsoviewed}) are familiar examples of such similarity-based recommenders. Items are recommended to a user on the basis of how \emph{similar} they are to item the user is currently browsing.

\begin{SCfigure}
    \fbox{\includegraphics[width=0.7\textwidth]{fig/alsoviewed.png}}
    \caption{An example of neighborhood-based recommendation (`People who viewed X also viewed Y').
    \label{fig:alsoviewed}}
\end{SCfigure}

For such a recommender to be effective depends on choosing an appropriate similarity function. The similarity function in \Cref{fig:alsoviewed} is based on view data (presumably clicks); but even then, by what metric should we consider patterns of clicks to be `similar'? Do we literally just count the number of users who have clicked on both items? Or do we need some kind of normalization? Or should we consider temporal recency?

Appropriately designing such similarity functions, and recommending on the basis of such similarity, is the task of so-called \emph{neighborhood-based} recommender systems---items are recommended due to being in the \emph{neighborhood} of (i.e.,~similar to) other items.

\begin{figure}
\caption{Memory-based and model-based recommender systems \label{box:modelBased}}
The neighborhood-based recommendation approaches we study in \Cref{sec:neighborhood} make recommendations by writing down similarity functions that take as input user and item interaction histories. There is no \emph{model} (i.e.,~no parameters) associated with each user or item; the original data is used as an input to the similarity functions. Such systems are referred to as \emph{memory-based} recommenders.
In contrast, the techniques in \Cref{sec:modelBased} learn \emph{representations} of each user and item, such that the original data isn't used directly when making a prediction at test time. Such systems are referred to as \emph{model-based} recommenders.
%TODO: Confusing, not written great
\end{figure}

\subsection{Defining a Similarity Function}

Defining a recommender like the one in \Cref{fig:alsoviewed} essentially requires that we define a similarity function among items:
\begin{equation}
    \mathrm{sim}(i,j),
\end{equation}
and then given a query item $i$, recommend a set of items $j$ that maximize the given similarity.

Let's consider a small toy example, with four items, and sets of users (or rather user IDs) who have consumed each:
\begin{equation}
    \begin{tabular}{l}
    $U_1 = \lbrace 1, 3, 4, 8, 12, 15, 17, 24, 35, 39, 41, 43 \rbrace$\\
    $U_2 = \lbrace 2, 3, 4, 5, 9, 12, 13, 16, 19, 24, 27, 31 \rbrace$\\
    $U_3 = \lbrace 4, 5, 9, 12 \rbrace$\\
    $U_4 = \lbrace 4, 9 \rbrace$
    \end{tabular}
    \label{eq:simToy}
\end{equation}
(recall that in our notation $U_1$ represents the set of users who have bought item 1).
%\todo{check this}
Na\"ively, we might assume that a recommender like the one depicted in \Cref{fig:alsoviewed} is simply counting the number of users who purchased both items in common. In our set notation this would be:
\begin{equation}
    \mathrm{sim}(i,j) = | U_i \cap U_j |.
\end{equation}
Computing some similarities under this model we would find:
\begin{equation}
    \mathrm{sim}(1, 2) = 4; \quad \mathrm{sim}(2, 3) = 3; \quad \mathrm{sim}(3, 4) = 2;  \quad \text{etc.}
\end{equation}
i.e.,~we would rate items 1 and 2, or items 2 and 3, as being \emph{more similar} than items 3 and 4.

We should examine whether these relative scores seem reasonable. Items 1 and 2 are \emph{popular} items, which most users did \emph{not} purchase in common; whereas items 3 has \emph{half} of its users in common with item 4. If we were to build recommenders on this basis, we might recommend (for example) a popular album as being highly similar to a popular pair of jeans, simply on the basis that they have many users in common. In general, such a system would tend to identify popular items (such as items 1 and 2 in \Cref{eq:simToy}) as being similar. `Niche' items with fewer associated purchases (such as items 3 and 4 in in \Cref{eq:simToy}) would rarely be recommended.

In most cases, this is not the outcome we want; such a system would make generic recommendations of popular items, that likely would not seem specific to the context of a given query item. Presumably, we should improve our similarity function so that it has some appropriate normalization to account for item popularity, as we will see below.

This toy example is intended to demonstrate that `similarity' is not something easy to define, and that different definitions have implicit assumptions with non-obvious consequences.

\subsection{Jaccard Similarity}
\label{sec:jaccard}

Our first attempt at correcting the above issue is to normalize similarity scores in a way that considers the popularity of each item. The \emph{Jaccard Similarity}, or `intersection over union,' does so by computing
\begin{equation}
    \mathit{Jaccard}(i,j) = \frac{| U_i \cap U_j |}{| U_i \cup U_j |}.
    \label{eq:jaccard}
\end{equation}
This similarity function is perhaps best visualized by a Venn diagram such as that in \Cref{fig:venn}. The Jaccard similarity takes a value between 0 (when $U_i$ and $U_j$ do not overlap at all, and thus have no intersection) and 1 (when the intersection is equal to the union, i.e.,~the items were consumed by \emph{exactly} the same set of people).

%TODO: Make this look better
\begin{SCfigure}
%\fbox{\includegraphics[width=0.3\textwidth]{fig/venn1.pdf}}
\begin{tikzpicture}[fill=gray]
% left hand
\scope
\clip (1,0) circle (1);
\fill (0,0) circle (1);
\endscope
% right hand
\scope
\clip (0,0) circle (1);
\fill (1,0) circle (1);
\endscope
% outline
\draw (0,0) circle (1) (0,1)  node [text=black,above] {$U_i$}
      (1,0) circle (1) (1,1)  node [text=black,above] {$U_j$};
\end{tikzpicture}\hspace{1mm}
\begin{tikzpicture}[fill=gray]
% left hand
\scope
% \clip (1,0) circle (1);
\fill (0,0) circle (1);
\endscope
% right hand
\scope
% \clip (0,0) circle (1);
\fill (1,0) circle (1);
\endscope
% outline
\draw (0,0) circle (1) (0,1)  node [text=black,above] {$U_i$}
      (1,0) circle (1) (1,1)  node [text=black,above] {$U_j$};
\end{tikzpicture}
\caption{The similarity between two items $i$ and $j$ can be computed in terms of the intersection (left) and union (right) between the sets of users $U_i$ and $U_j$ who have consumed each. \label{fig:venn}}
\end{SCfigure}

To demonstrate the Jaccard similarity in action, let's consider computing the similarity among items in terms of past purchases from \emph{Amazon.com}. 
%\todo{Make sure company names and stuff are italicized consistently}
We'll consider Amazon's publicly-available dataset of around 900,000 reviews from the \emph{Musical Instrument} category.\footnote{Available from \url{https://s3.amazonaws.com/amazon-reviews-pds/tsv/index.txt}}

We first build some data structures to store the sets of items consumed by each user (or the sets of users who have consumed each item), i.e.,~$I_u$ and $U_i$:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
usersPerItem = defaultdict(set)
itemsPerUser = defaultdict(set)

for d in dataset:
    user,item = d['customer_id'], d['product_id']
    usersPerItem[item].add(user)
    itemsPerUser[user].add(item)
\end{lstlisting}
\end{minipage}
We can also implement the Jaccard similarity straightforwardly:
%\todo{fix indentation level for all code}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
def Jaccard(s1, s2):
    numerator = len(s1.intersection(s2))
    denominator = len(s1.union(s2))
    return numerator / denominator
\end{lstlisting}
\end{minipage}
Now, \emph{recommendation} consists of finding the items with the highest Jaccard similarity compared to some given query (i.e.,~``people who bought X also bought Y''):
%\todo{comment code throughout}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
def mostSimilar(i, K): # Query item i, and number of results K to return
    similarities = []
    users = usersPerItem[i] # Users who have purchased i
    for j in usersPerItem: # Compute similarity against each item
        if j == i: continue
        sim = Jaccard(users, usersPerItem[j])
        similarities.append((sim,j))
    similarities.sort(reverse=True) # Sort to find the most similar
    return similarities[:K]
\end{lstlisting}
\end{minipage}

Finally, let's examine some recommendations, e.g.~of the \emph{`AudioQuest LP record clean brush'} (product ID \emph{B0006VMBHI}). The 5 most similar items (i.e., \texttt{mostSimilar('B0006VMBHI', 5)}) are:
\begin{eqnarray*}
\begin{tabular}{l}
\emph{Shure SFG-2 Stylus Tracking Force Gauge} \\
\emph{Shure M97xE High-Performance Magnetic Phono Cartridge} \\
\emph{ART Pro Audio DJPRE II Phono Turntable Preamplifier} \\
\emph{Signstek [...] Long-Playing LP Turntable Stylus Force Scale Gauge Tester} \\
\emph{Audio Technica AT120E/T Standard Mount Phono Cartridge}
% \emph{Technics: 45 Adaptor for Technics 1200 (SFWE010)} \\
% \emph{GruvGlide GRUVGLIDE DJ Package} \\
% \emph{STANTON MAGNETICS Record Cleaner Kit} \\
% \emph{Shure M97xE High-Performance Magnetic Phono Cartridge} \\
% \emph{Behringer PP400 Ultra Compact Phono Preamplifier}
\end{tabular}
\end{eqnarray*}
All of the recommended items are also related to record players (which make up only a fraction of items in the category), which seem semantically reasonable given the query.

And in just a few lines of code, and using a (reasonably large) real-world dataset, we have implemented our first recommender system! Our solution is simple (and our implementation is \emph{very} inefficient\footnote{In particular it is not necessary to iterate over \emph{all} items, but rather one can quickly compute a candidate set of only those items that could potentially have a non-zero Jaccard coefficient; see Exercise \ref{?}.}), but nevertheless quickly produced reasonable recommendations. Perhaps not surprisingly, these simple types of similarity-based recommendations drive many of the most high-profile recommender systems on the Web (see \Cref{sec:caseStudyAmazonRec}).
%\todo{hastily written, weak prose}

%TODO: Above code is not efficient

\subsection{Cosine Similarity}
\label{sec:cosine}

The Jaccard similarity captures our intuition about what items ought to be `similar' to each other, but is only defined if interactions are represented as \emph{sets}. We would like more nuanced similarity measures for data where feedback is associated with each interaction; for example we might not regard two users as `similar' if both had watched the \emph{Harry Potter} movies, in the event that one of them had liked the series and the other had disliked it.

The \emph{Cosine Similarity} achieves this by representing users' (or items') interaction histories in terms of \emph{vectors} rather than in terms of \emph{sets}. An example is shown in \Cref{fig:cosine} in which we have three items ($i_1$, $i_2$, and $i_3$) and two users ($u_1$ and $u_2$) who have each interacted with two of them. 

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/cosine.pdf}
\caption{The Cosine Similarity is defined in terms of the angle between two vectors, here describing users $u_1$ and $u_2$. \label{fig:cosine}}
\end{SCfigure}

In our previous (set) representation we would write $I_{u_1} = \lbrace i_2, i_3 \rbrace$ and $I_{u_2} = \lbrace i_1, i_3 \rbrace$ to describe the sets of items that $u_1$ and $u_2$ have interacted with. In vector representation we simply represent the users as $u_1 = (0,1,1)$ and $u_2 = (1,0,1)$; note that these vectors are equivalent to rows of our interaction matrix $R$ (\Cref{eq:interactionMatrix}), i.e.,~$R_{u_1}$ and $R_{u_2}$.
%\todo{Sloppy notation. Using a vector to index a matrix}

The Cosine Similarity (in this case between two users $u_1$ and $u_2$) is now defined in terms of the \emph{angle} between the vectors $u_1$ and $u_2$. Recall that the angle between two vectors $a$ and $b$ is defined as:
\begin{equation}
    \theta = \mathrm{Cos}^{-1} \left( \frac{a \cdot b}{|a|\cdot |b|} \right); \quad \text{or} \quad \mathrm{Cos}(\theta) = \frac{a \cdot b}{|a|\cdot |b|}.
\end{equation}
The angle $\theta$ measures the extent to which the two vectors point in the same direction; in the case of interaction data, the angle will range between \ang{0} (if the two users have interacted with exactly the same set of items) and \ang{90} (if the interaction vectors are orthogonal, i.e.,~if the users have interacted with non-overlapping sets of items). The actual cosine similarity 
%\todo{consistent capitalization}
is the cosine of this angle, e.g.~between two users $u$ and $v$ is:\footnote{Again it can be straightforwardly defined for \emph{item} similarity by interchanging users and items.}
\begin{equation}
    \text{Cosine Similarity}(u,v) = \frac{I_u \cdot I_v}{|I_u| \cdot |I_v|}.
    \label{eq:cosineSim}
\end{equation}
For (binary) interaction data the cosine of the angle is now between 1 (when the angle is zero, and the interactions are identical) and 0 (when the interactions are orthogonal).

It is instructive to compare \Cref{eq:jaccard,eq:cosineSim}. 
In the case of binary interaction data, both expressions take values of 1 when the interactions are identical, and 0 when the interactions are non-overlapping.
The numerator $I_u \cdot I_v$ in \Cref{eq:cosineSim} is equivalent to $|I_u \cap I_v|$, as in \Cref{eq:jaccard}. The two differ only in their denominators, both of which are essentially forms of normalization based on the size of the sets $I_u$ and $I_v$ (and both demoninators will take the same value when $I_u$ and $I_v$ are equal).

Of course the cosine similarity is more interesting once we consider numerical interactions, i.e.,~interactions associated with \emph{feedback}, rather than just binary (0/1) data. For example, consider data where each interaction is associated with a `thumbs-up' or `thumbs-down' rating. We might represent this via an interaction matrix $R$ such that $R_{u,i} \in \lbrace -1, 0, 1 \rbrace$ (where $-1/1$ indicates thumbs-up/thumbs-down, and $0$ indicates an item the user hasn't interacted with).

In this case, the Jaccard similarity would not be well-defined. But the cosine similarity can still be computed on rows (or columns) of $R$. An example is shown in \Cref{fig:cosine2}. Here, user $u_1$ has interacted with two items ($i_2$ and $i_3$), and liked both; $u_2$ has interacted with the \emph{same} items, but disliked both. The two user vectors now point in opposite directions, i.e.,~they have an angle of \ang{180} and a cosine similarity of -1.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/cosine2.pdf}
\caption{The Cosine Similarity for two users who rated the same items, but with opposite sentiment polarity. \label{fig:cosine2}}
\end{SCfigure}

%Our code above for the Jaccard similarity can be straightforwardly modified 
With some effort we can adapt our code above for the Jaccard similarity
to implement the cosine similarity. Here we use an auxiliary data structure (\texttt{ratingDict}) that retrieves ratings for a given user/item pair:
%(\Cref{exercise:cosine}).
%\todo{fix or delete exercise}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def Cosine(i1, i2):
    # Between two items
    inter = usersPerItem[i1].intersection(usersPerItem[i2])
    numer = sum([ratingDict[(u,i1)]*ratingDict[(u,i2)] for u in inter])
    norm1 = sum([ratingDict[(u,i1)]**2 for u in usersPerItem[i1]])
    norm2 = sum([ratingDict[(u,i2)]**2 for u in usersPerItem[i2]])
    denom = math.sqrt(norm1) * math.sqrt(norm2)
    if denom == 0: return 0 # If one of the two items has no ratings
    return numer / denom
\end{lstlisting}
\end{minipage}
%\todo{test code}

Doing so (for the same query item), the top recommendation remains the same (Shure SFG-2 Stylus Tracking Force Gauge). Among the next few recommendations, there are many ties (i.e.,~identical cosine similarities); upon inspection these turn out to be items with only a single (overlapping) interaction. In this case such items are preferred by the cosine similarity (and not the Jaccard) since the denominator grows quickly for items with many associated interactions (whereas the union term in \Cref{eq:jaccard} grows more slowly, assuming many of the interactions overlap).
%\todo{Results look a bit funny, check.}

\paragraph{Which similarity metric is `better'?}
%The above argument about the difference between the Jaccard and cosine similarity is somewhat imprecise. 
In the above example the Jaccard similarity seemed to work `better' than the Jaccard similarity, but our argument about the difference between the two is somewhat imprecise.
Note that this argument largely applies to this specific dataset (or even the specific query item we chose). Ultimately, these similarity measures are essentially \emph{heuristics}; whether one is `better' than another depends on our own assumptions and intuition about what similarity ought to mean. This is in contrast to the machine learning approaches we saw in previous chapters in which we had a specific objective (i.e.,~a measure of success) that we were trying to optimize. We'll revisit this question when we examine model-based recommender systems in \Cref{sec:modelbased}.

\subsection{Pearson Similarity}
\label{sec:pearsonSim}

We motivated the Jaccard similarity by considering binary interaction data (i.e.,~sets), and the cosine similarity by considering polarized interactions like `thumbs-up's and `thumbs-down's. But how would these similarity measurements operate on numerical feedback scores such as star ratings (as in \Cref{eq:interactionMatrix})?

Consider the users represented in \Cref{fig:pearson1} (left).
%\todo{Most of these 3d plots would look better with shorter $i_1$ axis.}
Here two users have rated the same items ($i_2$ and $i_3$); user $u_1$ rated them 3 and 5 stars (respectively); user $u_2$ rated them 5 and 3.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/pearson1.pdf}%
\includegraphics[scale=0.75]{fig/pearson2.pdf}
\caption{Pearson Correlation. Two users have rating vectors that point roughly in the same direction (left); after subtracting the average from each, they point in opposite directions (right). \label{fig:pearson}}
\end{SCfigure}

According to the Jaccard similarity (based only on interactions), we would consider the two users to be identical (Jaccard similarity of 1); according to the cosine similarity, we would regard them as being very similar, since the angle between the two vectors is small. However one could argue that these users are polar opposites of each other: if we consider 5 stars to be a \emph{positive} rating and 3 stars is a \emph{negative} rating, then these two users indeed have opposite opinion polarity.

Our definition of the cosine similarity does not account for this interpretation, essentially because it depends on the interactions already having explicitly positive or negative polarity. To might correct this by appropriately normalizing our ratings: if we subtract the average for each user (4 stars for both $u_1$ and $u_2$), we find that their ratings are each 1 star above or below their personal average. After doing so, the example becomes very similar to the one from \Cref{sec:cosine} (see \Cref{fig:pearson}, right).

This is essentially the idea captured by the Pearson Correlation. %\todo{again, check capitalization} 
The \emph{Pearson Correlation Coefficient} is a classical measurement for assessing the relationship between two variables, i.e.,~whether they trend in the same direction, regardless of scale and constant differences between them. The Pearson Correlation between two vectors $x$ and $y$ is defined as
\begin{equation}
    \text{Pearson Correlation}(x,y) = \frac{\sum_{i=1}^{|x|} (x_i - \bar{x})(y_i - \bar{y})}{\sqrt{\sum_{i=1}^{|x|}  (x_i - \bar{x})^2}\sqrt{\sum_{i=1}^{|y|} (y_i - \bar{y})^2}}.
    \label{eq:pearsonCC}
\end{equation}
Compare this definition to that of the cosine similarity in \Cref{eq:cosineSim}: the only difference is that from each measurement we subtract the mean ($\bar{x}$ or $\bar{y}$) of the corresponding vector.

When applying this concept to rating data, we should be careful not to regard unobserved ratings (i.e.,~missing values of $R_{u,i}$ in \Cref{eq:interactionMatrix}) as zeros---doing so would distort our estimate of the user mean. Thus we might define the similarity between two users $u$ and $v$ (or similarly, items) only in terms of items they have 
both 
interacted with:
\begin{equation}
    \text{Pearson Similarity}(u,v) =
    \frac
    {
      \sum_{i \in I_u \cap I_v}
        (R_{u,i} - \bar{R_u})
        (R_{v,i} - \bar{R_v})
    }
    {
      \sqrt{\sum_{i \in I_u \cap I_v} (R_{u,i} - \bar{R_u})^2}
      \sqrt{\sum_{i \in I_u \cap I_v} (R_{v,i} - \bar{R_v})^2}
      %\sqrt{\sum_{i \in I_u} (R_{u,i} - \bar{R_u})^2}
      %\sqrt{\sum_{i \in I_v} (R_{v,i} - \bar{R_v})^2}
    }.
    \label{eq:pearsonSim}
\end{equation}
%\todo{Think about putting this argument back in}
Note that our choice to define the Pearson Similarity by considering only \emph{shared} items is somewhat arbitrary; we could instead have considered \emph{all} items rated by each user in the denominator. Using our definition (which appears in e.g.~\cite{sarwar2001item}), we regard users as maximally similar if they have rated shared items in the same way; if we considered all items rated by each user in the denominator, we would regard users as less similar if they had also rated some different items. Remember that these similarity functions are merely heuristics---neither option should be considered more `correct,' but rather we should choose the definition that suits our intuition (or generates the most satisfactory results) in a particular situation.
%\todo{long argument to make a not very important point! Think whether to include this.}

%\done{included code examples and results}

%\todo{Bullets don't look great, maybe use a {description} environment or similar}
\begin{figure}
\caption{Summary of Similarity Measures. \label{box:sumSim}}
Our comparison of the Jaccard, Cosine, and Pearson similarity can be summarized as follows:
\begin{itemize}
    \item The \emph{Jaccard Similarity} computes the similarity between \emph{sets}. The basic idea is to find items that have been purchased in common by many users (or similarly, users who have purchased many items in common); the set union is used to normalize the quantity, so that the measure does not overly favor large sets.
    \item The \emph{Cosine Similarity} instead represents interactions as vectors (basically, rows or columns of our interaction matrix $R$). Similarity is then computed in terms of the angle between vectors for two items (or users). This definition allows similarity to be computed for numerical interaction data, especially if a polarity (i.e.,~positive or negative) is associated with each interaction.
    \item Finally, the \emph{Pearson Similarity} was motivated by the idea that numerical feedback may need to be properly calibrated in order to associate a polarity to each score. For instance, a rating of `3.5' might be positive for one user but negative for another. The Pearson Similarity calibrates this polarity simply by subtracting the average for each item (or user); after this calibration the definition is similar to that of the cosine similarity.
\end{itemize}
\end{figure}

Our code for the cosine similarity above can easily be adapted to implement the Pearson Similarity (noting the details above). Here we have an additional data stucture (\texttt{itemAverages}) recording the mean rating for each item:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def Pearson(i1, i2):
    # Between two items
    iBar1,iBar2 = itemAverages[i1], itemAverages[i2]
    inter = usersPerItem[i1].intersection(usersPerItem[i2])
    numer = 0
    denom1 = 0
    denom2 = 0
    for u in inter:
        numer += (ratingDict[(u,i1)] - iBar1)*(ratingDict[(u,i2)] - iBar2)
    for u in inter: # Alternately could sum over usersPerItem[i1]/[i2]
        denom1 += (ratingDict[(u,i1)] - iBar1)**2
        denom2 += (ratingDict[(u,i2)] - iBar2)**2
    denom = math.sqrt(denom1) * math.sqrt(denom2)
    if denom == 0: return 0
    return numer / denom
\end{lstlisting}
\end{minipage}

Fitting the Pearson Similarity given the query item from \Cref{sec:jaccard} does not produce particularly satisfactory results; 
%the similar items come from a broad range of categories that do not seem to be closely related to the query item.
using $U_i \cap U_j$ in the denominator of \Cref{eq:personSim} results in many items with a similarity of 1.0 (usually just due to a single overlapping interaction); using $U_i$ and $U_j$ separately in the denominator of \Cref{eq:personSim} generates more meaningful results, though they come from a broad category of items that do not seem closely related.

Possibly these results are unsatisfactory simply because ratings of (e.g.)~a record cleaning brush are not due to factors that meaningfully transfer to other items. One likely purchses a record cleaning brush for its utility, rather than because of their personal preference toward such items. If variability in ratings is primarily due to build quality, or effectiveness (for example), then the Pearson Similarity might identify other items with `similar' build quality or effectiviness, but those may not be semantically similar items. In this particular example, the Jaccard similarity---which defines similarity in terms of \emph{what} was purchased---seems more appropriate than the Pearson Similarity, which defines similarity in terms of preferences.

Again though, possibly this measure is simply not suitable for this dataset or this query item. Let's try again on another dataset, this time from the Amazon \emph{Video Games} category. Given the query \emph{One Piece: Pirate Warriors}, the five most similar items in terms of Pearson Similarity\footnote{Again using $U_i$ and $U_j$ separately in the denominator} are:
\begin{eqnarray*}
\begin{tabular}{l}
\emph{Full Metal Alchemist: The Broken Angel}\\
\emph{Monster Rancher 4}\\
\emph{FINAL FANTASY X X-2 HD Remaster}\\
\emph{BlazBlue: Continuum Shift EXTEND Limited Edition}\\
\emph{Killzone 3}
(etc.)
\end{tabular}
\end{eqnarray*}
These recommendations look more reasonable. In addition to being for similar platforms (e.g.~PlayStation) most are reasonably similar in terms of genre and style (e.g.~Japanese, based on anime, etc.). Seemingly, for this type of time, features like style and genre better explain variation in ratings, making the Pearson Correlation more effective.

Finally, these similarity measures needn't be used so directly for recommendations as we have done here (i.e.,~simply retrieving the most similar item given a query). In practice they might simply be subroutines that guide more complex algorithms. For example, to recommend items to a user we might first find similar users, and recommend items that many of those users liked, rather than simply relying on item-to-item similarity directly (see e.g.~\Cref{sec:caseStudyAmazonRecs}).
%\todo{weak prose above}

%%\todo{Put this under some section heading?}
%Moved above
%Note that the memory-based recommendation approaches above are essentially \emph{heuristics} for determining whether items (or users) are similar to each other; each makes different assumptions about what `similarity' might mean, each of which might be appropriate given different datasets or goals.

\subsection{Using Similarity Measurements for Rating Prediction}

In \Cref{sec:modelbased}, we will contrast the similarity-based recommendation approaches above with machine learning (or `model-based') approaches which directly seek to predict ratings as accurately as possible.

However these two goals (measuring similarity versus predicting ratings) are not at odds with each other, and indeed one can use a measure of similarity as a means of predicting ratings.

The essence of such an approach is that the rating a user will give to an item can be estimated from ratings that user has given to \emph{similar} items (again, for some appropriate definition of `similarity'). One such definition (from \cite{sarwar2001item}) predicts the rating as a weighted sum of other items the user has rated:
\begin{equation}
    r(u,i) = \frac{\sum_{j \in I_u} R_{u,j} \cdot \mathrm{Sim}(i,j)}{\sum_{j \in I_u} \mathrm{Sim}(i,j)},
    \label{eq:simPredict}
    %\todo{Check summations in all of these equations. Check that subscripts of R are right (had to fix some)}
\end{equation}
where $\mathrm{Sim}(i,j)$ could be any item-to-item similarity function such as those above. Note here that $r(u,i)$ is a prediction whereas $R_{u,i}$ is a historical rating.

The intuition behind the above equation is simply that the most similar items should be the most relevant when predicting future ratings, so the user's past ratings of those items are given the highest weights. Again though this is just a heuristic for predicting ratings and could be defined differently. For example we could write the same definition in terms of user similarity:
\begin{equation}
    r(u,i) = \frac{\sum_{v \in U_i} R_{u,j} \cdot \mathrm{Sim}(u,v)}{\sum_{v \in U_i} \mathrm{Sim}(u,v)},
    \label{eq:simPredict2}
\end{equation}
or, we, could possibly improve performance slightly by weighting deviations from the average rating, rather than ratings directly:
\begin{equation}
    r(u,i) = \bar{R_i} + \frac{\sum_{j \in I_u} (R_{u,j} - \bar{R_j}) \cdot \mathrm{Sim}(i,j)}{\sum_{j \in I_u} \mathrm{Sim}(i,j)},
    \label{eq:simPredict3}
\end{equation}

Using our video game data from \Cref{sec:pearsonSim}, and following the prediction function from \Cref{eq:simPredict3} (with the Jaccard Similarity as our similarity function), the Mean Squared Error of predicted ratings compared to the true labels is 1.786,
%1.9150345376505316 - simPredict
%
%1.7861436807136897 - simPredict3
compared to 1.838
%1.838224481118372
when always predicting the mean.

Code to implement the rating prediction model of \Cref{eq:simPredict3} is included below. Here we use the Jaccard similarity, though any item-to-item similarity metric could be used in its place:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def predictRating(user,item):
  ratings = [] # Collect ratings over which to average
  sims = [] # and similarity scores
  for d in reviewsPerUser[user]:
    j = d['product_id']
    if j == item: continue # Skip the query item
    ratings.append(d['star_rating'] - itemAverages[j])
    sims.append(Jaccard(usersPerItem[item],usersPerItem[j]))
  if (sum(sims) > 0):
    weightedRatings = [(x*y) for x,y in zip(ratings,sims)]
    return itemAverages[item] + sum(weightedRatings) / sum(sims)
  else:
    # User hasn't rated any similar items
    return ratingMean
\end{lstlisting}
\end{minipage}

%\todo{In various code blocks, have line numbers follow across blocks}
%\todo{Consistent indentation for code}

\section{Case Study: Amazon.com Recommendations}
\label{sec:amazonCaseStudy}

In a 2003 paper \cite{linden2003amazon}, researchers described the techniques underlying \emph{Amazon.com}'s recommendation technology. The paper described systems similar to that in \Cref{fig:alsoviewed}, e.g.~``Customers who bought items in your shopping card also bought.''

The first recommendation method the paper describes is based on the Cosine Similarity (\Cref{sec:cosine}). Interestingly, cosine similarity is defined between \emph{users}, rather than between items as we did in \Cref{sec:jaccard}; the goal is then to recommend items that have previously been purchased by similar customers. The paper discusses the issues of scaling this type of similarity computation to the large number of \emph{Amazon} users, and discusses an alternative strategy to \emph{cluster} users based on a user-to-user similarity metric; similar customers to a given user by determining the user's cluster membership (which is cast as a classification problem).

Although \cite{linden2003amazon} goes into little detail about the specifics of what is implemented on Amazon.com, their work does stress the key point that real-world, large-scale recommenders need not be based on complex models. Rather, primary considerations including building models that are simple but scale well.

%Cite the old amazon recommendation paper, and `Two Decades of Recommender Systems at Amazon.com'

Following \cite{linden2003amazon}, a follow up paper was published describing more modern recommendation techinques on Amazon.com \cite{smith2017two}.
%\todo{italics in this section}
The paper starts by describing minor modifications to the algorithms described above. For instance, they describe an item-to-item based approach which is more reminiscent of that in \Cref{sec:jaccard}, and describe how most of the computation for this type of problem can be done offline. \cite{smith2017two} also stresses the importance of choosing a good similarity heuristic, and discusses some strategies for doing so. Finally, \cite{smith2017two} discusses the importance of considering temporal factors when designing recommenders, which is our main focus in \Cref{chap:temporalSequential}.


\section{Model-based Approaches to Recommendation}
\label{sec:modelbased}

In contrast to `memory-based' recommendation approaches, \emph{model-based} approaches seek to learn parameterized representations of users and items, so that recommendations can be made in terms in terms of the learned parameters. Model-based approaches are typically cast in terms of supervised learning, so that the goal is to predict ratings, purchases, clicks (etc.)~as accurately as possible.

\begin{figure}
\caption{Memory-based vs.~Model-based Approaches}
There are a variety of reasons why one might choose a model-based or memory-based approach. We summarize a few of the advatages and disadvantages as follows:
\begin{description}
  \item[Training and Inference Complexity] Model-based approaches often require (expensive) offline training; on the other hand, once trained, recommendations can potentially be retrieved quickly, e.g.~by retrieving a nearest neighbor or a maximum inner product in parameter space. In contrast, 
  %TODO: Finish
  \item[Interpretability] Often, simple recommendations such as those in \Cref{fig:alsoviewed} may be preferable simply because they are easy to explain to a user. In contrast, machine learning-based recommendations may make users uncomfortable due to their `black box' nature (see \Cref{sec:interpretability}).
  \item[Accuracy] Model-based systems are appealing because they directly optimize a desired error measure. On the other hand, error measures that are tractible may not be those that relate meaningfully to user satisfaction, and may distract from qualitative improvements.
\end{description}
\end{figure}

%TODO: Other methods
%SLIM
%FISM
%Some sort of deep learning based model


\subsection{The Netflix Prize}
%\todo{Put this subsection somewhere, or make it a case study}

In 2006, \emph{Netflix} released a dataset of 100,000,000 movie ratings (across 17,770 users and around 480,000 users). Their dataset took exactly the form described in \Cref{sec:setup}, i.e.,~it consisted purely of (user, item, rating, timestamp) tuples. Associated with the dataset was a competition \cite{bennett2007netflix} to reduce the RMSE (on a test set with withheld ratings) by 10\% compared to Netflix's existing solution. The first team to do so would win a \$1,000,000 prize. 


The history of the competition is itself fascinating. Early leaders joined forces to develop ensemble approaches, and the winning teams were nearly tied in a nail-biting finish. The competition also led to a broader discussion around the value of such high-profile competitions, as well as the question of whether narrowly reducing a mean-squared error actually improves recommendations. Finally it also led to a lawsuit against Netflix following de-anonymization of the competition data \cite{narayanan2006break}.\footnote{This too is an interesting story, as the competition data, with anonymized user and item IDs, at first glance appears to be sufficiently anonymized.}
%\todo{Could discuss further. Also include e.g. whether monolithic models can really be implemented}

But other than the history surrounding the Netflix Prize, the dataset and high-profile competition spawned a great deal of research on recommender systems in general, and in particular the specific problem of rating prediction.
%\todo{rest of this sentence}
and in particular, solutions based on matrix factorization.

\begin{figure}
\caption{Lessons from the Netflix Prize}
%TODO
Add text on ``Lessons from the Netflix Prize Challenge'', maybe in box.
\end{figure}




\subsection{Matrix Factorization}

The basic assumption made by model-based recommenders is that there is some underlying \emph{structure} among the interactions we are trying to predict. Put differently, model-based recommender systems are essentially a form of dimensionality reduction.
%\todo{This and below could be in the start of the section, or the subsection heading could be below}

In simple terms, we assume that users' opinions, or the properties of the items they consume, can be efficently summarized. Do you tend to like action movies (and is this an action movie)? Do you tend to enjoy movies with a high budget, certain actors, or a long runtime? To the extent that purchases, clicks, or ratings can be explained by such factors, the goal of model-based recommendation is to discover them.

Considering data of the form in \Cref{sec:setup}, this seems a difficult process: we have no knowledge of the necessary \emph{features} that would be needed to discover these important factors (i.e.,~we don't know which movies are action movies, which movies have a long runtime, etc.). But surprisingly one can still uncover these underlying dimensions. Consider the interaction (e.g.~click) data depicted below:
\begin{equation}
    R = 
    \begin{tabular}{cc}
    $\left[ \begin{tabular}{ccccc}
    1 & $\cdot$ & 1 & $\cdot$ & $\cdot$\\
    1 & 1 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 1 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & $\cdot$ & $\cdot$ & 1 & 1\\
    $\cdot$ & $\cdot$ & $\cdot$ & 1 & $\cdot$\\
    \end{tabular} \right]$
    &
    \hspace{-6mm}
        \begin{tabular}{r}
        $u_1$\\
        $u_2$\\
        $u_3$\\
        $u_4$\\
        $u_5$
        \end{tabular}\\
        
        \begin{tabular}{ccccc}
        \hphantom{1}$i_1$ &
        \clap{$i_2$}\hphantom{1} &
        \clap{$i_3$}\hphantom{1} &
        \clap{$i_4$}\hphantom{1} &
        \clap{$i_5$}\hphantom{1}
        \end{tabular}
    \end{tabular}
        \label{eq:interactionMatrix2}
\end{equation}
The matrix appears to decompose roughly into two `blocks': if we wrote
\begin{equation}
    \begin{tabular}{l}
      $\gamma_{i_1} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_2} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_3} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_4} = \lbrack 0,1 \rbrack$\\
      $\gamma_{i_5} = \lbrack 0,1 \rbrack$
    \end{tabular}; \quad
    \begin{tabular}{l}
      $\gamma_{u_1} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_2} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_3} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_4} = \lbrack 0,1 \rbrack$\\
      $\gamma_{u_5} = \lbrack 0,1 \rbrack$
    \end{tabular},
    \label{eq:interactionFactors}
\end{equation}
Then we could summarize the matrix $R$ in \Cref{eq:interactionMatrix2} by writing
\begin{equation}
    R_{u,i} = \gamma_u \cdot \gamma_i.
    \label{eq:matfac1}
\end{equation}
% \todo{Introduce the idea of a `user manifold' somewhere}
The two blocks in $R$ might conceivably correspond to some feature in the data, e.g.~male and female users who buy men's and women's clothing. If so, the values in \Cref{eq:interactionFactors} would correspond to genders, though we discovered these factors automatically simply because they \emph{summarized the structure of the matrix}, rather than needing to rely on observed features.
%\todo{Kind of hate the above example}.

\emph{Matrix Factorization} follows this same idea, again by looking for underlying structure that explains observed interactions.

Essentially, our goal is to describe a matrix in terms of lower-dimensional factors, i.e.,
\begin{equation}
    \underbrace{\left[ \begin{tabular}{ccc}
    \ & \ & \ \\
    \ & R & \ \\
    \ & \ & \ 
    \end{tabular} \right]}_{N_u \times N_i} = 
    \underbrace{\left[ \begin{tabular}{c}
    \ \ \ \\
    $\gamma_U$\\
    \  
    \end{tabular}
    \right]}_{N_u \times K} \times
    \underbrace{\left[
    \begin{tabular}{ccc}
    \parbox{1mm}{\vspace{5mm}} & $\gamma_I^T$ & \parbox{1mm}{\vspace{5mm}}\\
    \end{tabular} \right]}_{K \times N_i}.
\end{equation}
%\todo{Kind of hate the notation above. Could just use U and I for number of users and items?}
%\todo{Probably should use |U| and |I| for the number of users and items}
That is, we are assuming that the matrix $R$, of dimension $N_u \times N_i$ (the number of users times the number of items), can be approximated by a `tall' matrix $\gamma_U$ and a `wide' matrix $\gamma_I$. 
%$\gamma_U$ and $\gamma_I$ essentially summarize the users and items (respectively)
Now, a single entry $R_{u,i}$ can be estimated by taking the corresponding row of $\gamma_U$ and column of $\gamma_I$:
\begin{equation}
    R_{u,i} = \gamma_u \cdot \gamma_i,
    \label{eq:matfac2}
\end{equation}
as in \Cref{eq:matfac1}. 
%\todo{Kinda ugly that this is *exactly* the same as the previous equation}
$\gamma_u \in \mathbb R^K$ is now a latent vector that describes a user, and $\gamma_i \in \mathbb R^K$ describes an item.

Examples of such vectors are depicted in \Cref{fig:matfac1}. Intuitively, $\gamma_u$ might be thought of as describing the `preferences' of the user $u$, whereas $\gamma_i$ describes the `properties' of item $i$. Then, the user $u$ will like (e.g.~give a high rating or interact with) the item $i$ if their preferences are \emph{compatible} with the item's properties (i.e.,~they have a high inner product). The latent dimensions, $\gamma_{\cdot,1}$, $\gamma_{\cdot,2}$ (etc.)~now describe those latent factors that best explain variability in $R$. For example, if such a model were trained on the Netflix dataset they might measure the extent to which a movie is a comedy or a romance, or the quality of its special effects. Again though, these factors are \emph{latent} and are discovered purely so as to maximally explain the observed interactions; models based on the principle of matrix factorization are commonly referred to as \emph{latent factor models}.

%\todo{Mention SVD and cite Koren paper}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/matfac1.pdf}
\caption{Representation of a user $u$ and item $i$ in a latent-factor model. \label{fig:matfac1}}
\end{SCfigure}

\subsection{Fitting the Latent Factor Model}

So far we have described the intuition behind modeling interactions in terms of latent user and item factors, but have not yet described how to fit a model based on this principle. That is, we would like to choose $\gamma_U$ and $\gamma_I$ so as to fit the interaction data most closely, e.g.~by minimizing some loss such as the Mean Squared Error, following the setting of the Netflix Prize:
%\todo{Don't think I mentioned that the Netflix Prize used the MSE yet? Maybe not necessary}
\begin{equation}
    \argmin_{\gamma} \frac{1}{|R|} \sum_{(u,i) \in R} (f(u,i) - R_{u,i})^2,
\end{equation}
where $f(u,i)$ is our prediction function $f(u,i) = \gamma_u \cdot \gamma_i$.

Note that this would be straightforward if the matrix $R$ were fully observed (i.e.,~every user rated every item). In this case, we could rely on the Singular Value Decomposition (SVD). Briefly, this decomposition allows us to factorize a real $M\times N$ matrix according to
\begin{equation}
    R = U\Sigma V^T,
\end{equation}
where $U$, $V$, and $\Sigma$ eigenvectors and eigenvalues of $R^TR$ and $RR^T$. Critically, if we keep just the $K$ eigenvectors corresponding to the largest eigenvalues, this corresponds to the \emph{best possible rank $K$ factorization in terms of the MSE} \cite{golub1971singular}.

However, the Singular Value Decomposition is defined only for \emph{fully observed} matrices, rather than partially observed interactions as in $R$. Even if this could be addressed (e.g.~via a data imputation strategy on the missing values), it would not be practical to compute the SVD on a matrix that could potentially have millions of rows and columns.

Instead, we seek a solution based on gradient descent. When minimizing the Mean Squared Error, the solution is similar to the one we saw in \Cref{sec:gradient}. Note that as usual we should be careful to split interactions $(u,i) \in R$ into training, validation, and test sets, and to include a regularizer to avoid overfitting, as we describe below.

\subsubsection{User and Item Biases}

Before describing the gradient-descent-based solution in detail, we first suggest some steps to improve the solution.

Although a simple solution of the form $r(u,i) = \gamma_u \cdot \gamma_i$ seems to capture the types of interactions we want, it is difficult to regularize. Consider adding a simple $\ell_2$ regularizer such as
\begin{equation}
    \Omega(\gamma) = \lambda \left( \sum_{u = 1}^{|U|} \sum_{k = 1}^K \gamma_{u,k}^2 +
                                    \sum_{i = 1}^{|I|} \sum_{k = 1}^K \gamma_{i,k}^2
                            \right)
    \label{eq:latentreg1}
\end{equation}
%\todo{Never use the notation Omega elsewhere. Maybe use other places or drop here.
%\todo{Write |I| / |U| here but is N_i / N_u above. Like |I| better but I forget why I did N_i in the first place}
This regularizer (for large $\lambda$) will encourage the parameters to be close to zero; as such the predictions $\gamma_u \cdot \gamma_i$ will also be pushed toward zero, and the system will systematically underpredict ratings.

There are several ways this could be avoided. Trivially, we might simply subtract the mean ($\bar{R}$) from all ratings before training so that they are centered around zero. Alternately, recall as in \Cref{sec:?}  that we were careful to exclude the intercept term $\theta_0$ from our regularizer.
%\todo{That section doesn't exist yet}
Although our current model lacks such an intercept term, we can straightforwardly add one:
\begin{equation}
    r(u,i) = \alpha + \gamma_u \cdot \gamma_i.
\end{equation}
Note that we still regularize as in \Cref{eq:latentreg1}.

Although the offset term $\alpha$ corrects the problem of systematically underpredicting ratings, it retains a similar issue at the level of individual users or items. Again, the regularizer pushes $\gamma_u$ and $\gamma_i$ toward zero, and therefore pushes predictions toward $\alpha$. But individual users or items may tend to systematically give much higher (or lower) ratings than $\alpha$, meaning that our regularizer again encourages us to systematically under (or over) predict.

Again we correct for this by adding additional bias terms, this time at the level of individual users or items:
\begin{equation}
    r(u,i) = \alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i.
    \label{eq:MFscore}
\end{equation}
$\beta_u$ now encodes the extent to which user $u$'s ratings trend higher or lower than $\alpha$, and $\beta_i$ encodes the extent to which item $i$ tends to receive higher or lower ratings than $\alpha$.\footnote{Note that we are careful not to refer to $\alpha$ as an `average' rating, and indeed in general once we fit the model $\alpha \not= \bar{R}$, just as the offset $\theta_0$ is not the average $\bar{y}$ in a linear regression model.}

Adding these bias terms introduces an additional $|U| + |I|$ parameters to the model. Whether these terms should be included in the regularizer $\Omega$ is arguable: on the one hand they are similar to offset terms, which we would normally not regularize (for the same reason we do not regularize $\alpha$); on the other hand, failing to regularize them may lead to overfitting. In practice, the terms may simply be included in the regularizer:
\begin{equation}
    \lambda\Omega(\beta, \gamma) = \left(
    \sum_{u = 1}^{|U|} \beta_u^2 + \sum_{k = 1}^K \gamma_{u,k}^2 + 
    \sum_{i = 1}^{|I|} \beta_i^2 + \sum_{k = 1}^K \gamma_{i,k}^2
                      \right).
    \label{eq:latentreg2}
\end{equation}
Alternately one could regularize $\beta$ and $\gamma$ with different stregths (since we generally expect $\beta$ to have larger values:
\begin{equation}
    \lambda_1\Omega(\beta) + \lambda_2\Omega(\gamma) = \lambda_1 \left(
    \sum_{u = 1}^{|U|} \beta_u^2 + \sum_{i = 1}^{|I|} \beta_i^2 \right) + 
    \lambda_2 \left(
    \sum_{u = 1}^{|U|} \sum_{k = 1}^K \gamma_{u,k}^2 + 
    \sum_{i = 1}^{|I|} \sum_{k = 1}^K \gamma_{i,k}^2
                      \right),
    \label{eq:latentreg3}
\end{equation}
though doing so is difficult as it results in multiple regularization constants to tune.

\subsubsection{Gradient Update Equations}

Finally, the objective we wish to minimize (on a training set of interactions $\mathcal T$) is
\begin{eqnarray}
\text{obj}(\alpha; \beta; \gamma | \mathcal{T}) & = & \frac{1}{\mathcal T} \sum_{(u,i) \in \mathcal{T}} (r(u,i) - R_{u,i})^2 + \lambda \Omega(\beta, \gamma)\\
%\todo{check above}
    & = & \frac{1}{\mathcal{T}} \sum_{(u,i) \in \mathcal{T}} (\alpha + \beta_i + \beta_u + \gamma_i + \gamma_u - R_{u,i})^2 + \lambda \Omega(\beta, \gamma).
    \label{eq:MFObjective}
\end{eqnarray}
%\todo{Consistent notation for R_{(u,i)} in \mathcal{T} vs just (u,i) \in R or whatever.
Assuming the regularizer takes the form given in \Cref{eq:latentreg2}, the partial derivatives (for $\alpha$, $\beta_u$, and $\gamma_{u,k}$) are given by:
\begin{eqnarray*}
\frac{\partial \text{obj}}{\partial \alpha} & = & \sum_{(u,i) \in \mathcal{T}} 2(r(u,i) - R_{u,i})^2\\
\frac{\partial \text{obj}}{\partial \beta_u} & = & \sum_{i \in I_u} 2(r(u,i) - R_{u,i})^2 + 2\beta_u\\
\frac{\partial \text{obj}}{\partial \gamma_{u,k}} & = & \sum_{i \in I_u} 2\gamma_{i,k}(r(u,i) - R_{u,i})^2 + 2\gamma_{u,k}.
\label{eq:MFgradient}
\end{eqnarray*}
Note the change of summation in the last two terms: the derivative for user $u$ is based only on items $I_u$ that they consumed (in the training set). Derivatives for $\beta_i$ and $\gamma_{u,k}$ can be computed similarly and are left as an exercise (\Cref{ex:?}).

\paragraph{Other Considerations for Gradient Descent}
When we first introduced gradient descent in \Cref{sec:gradient}, we noted some potential issues in terms of local minima, learning rates etc.
It is worth revisiting some of those issues in light of the more complex model we are fitting here:
\begin{itemize}
    \item The problem in \Cref{eq:MFObjective} is certainly non-convex and has many local minima.\footnote{The proof can roughly be sketched as follows. The objective is smooth, and given any global optimum $\gamma_U$, $\gamma_I$, any permutation applied to both (i.e.,~$\gamma_U\pi$ and $\gamma_I\pi$) will result in an \emph{equivalent} local optimum.} Surprisingly though, this problem is not prone to `spurious' local optima, and if carefully implemented should converge to a global optimum \cite{ge2016matrix}.
    \item Nevertheless the problem is sensitive to initialization. For example one if multiple columns of $\gamma_U$ and $\gamma_I$ are initialized to the same value, they will have identical gradients and will remain in `lock step' during successive iterations. This can normally be avoided simply by randomized initialization.
    \item Rather than computing the full gradient as in \Cref{eq:MFgradient}, alternate approaches such as stochastic gradient descent, or alternating least squares, may converge faster or require less memory.\footnote{Alternating Least Squares notes that the optimization problem in \Cref{eq:MFObjective} has a closed form if either $\gamma_U$ or $\gamma_I$ is fixed; optimization proceeds by alternately fixing one term and optimizing the other.} See e.g.~\cite{bottou2010large,yu2012scalable}.
\end{itemize}
% The gradient descent strategy presented above
% Faster gradient descent (coordinate descent): \cite{yu2012scalable}


\section{Implicit Feedback and Ranking Models}

So far, our discussion of (model-based) recommender systems has focused on predicting real-valued outcomes such as ratings, using objectives based on the Mean Squared Error. That is, we have described model-based recommendation in terms of \emph{regression} approaches.

Just as we developed separate approaches for neighborhood-based recommendation when considering click, purchase, or rating data, here we consider how our regression-based approaches should be adapted to handle binary outcomes (such as clicks and purchases).

Na\"ively, we might imagine that we could adapt our regression-based approaches to handle binary outcomes much in the same way we that developed logistic regression. That is, we could pass the model output (\Cref{eq:MFscore}) through a sigmoid function, such that positive interactions are associated with high probabilities, and negative interactions are associated with low probabilities.

However, when dealing with click or purchase data, we should consider that items which haven't been clicked or purchased are not necessarily \emph{negative} interactions---in fact items that \emph{haven't} been clicked or purchased are exactly the ones we intend to recommend.

Several techniques have been proposed to handle recommendation in this context. Often this setting is referred to as \emph{one-class} recommendation, as only the `positive' class (clicks, purchases, listens, etc.)~is observed.

Also called implicit feedback recommendation

\subsection{Instance Re-weighting Schemes}

One category of methods for dealing with implicit feedback data attempts to reweight instances as having various `confidences' of being positive or negative.

\cite{hu2008collaborative} considers cases where positive instances are associated with `confidence' measures $r_{u,i}$, which could measure e.g.~the number of times a user listened to a song; in \cite{hu2008collaborative} the confidence values encode how many times a user has watched a particular program. Negative instances still have $r_{u,i} = 0$, such that the model essentially captures the fact that negative instances are necessarily associated with low confidence, whereas confidence may vary substantially among positive instances.

Ultimately, the goal is still to predict a binary outcome $p_{u,i}$, and the model is trained to predict
\begin{equation}
    p_{u,i} = \left\lbrace \begin{tabular}{cl}
    1 & if $r_{u,i} > 0$\\
    0 & otherwise
    \end{tabular}\right.
\end{equation}.
The form of the predictor is similar to that of \Cref{eq:MFObjective}, i.e.,~latent user and item factors are use to predict $p_{u,i}$ using a (regularized) Mean Squared Error. The main difference is that the MSE is weighted according to the confidence of each observation:
\begin{equation}
    \argmin_\gamma \sum_{(u,i) \in \mathcal T} c_{u,i}(p_{u,i} - \gamma_u \cdot \gamma_i)^2 + \lambda \Omega(\gamma),
    \label{eq:weightedMF}
\end{equation}
where $c_{u,i}$ is a weighting function associated with each observation, which ultimately is a monotone transform of $r_{u,i}$, e.g.
\begin{equation}
    c_{u,i} = 1 + \alpha r_{u,i}; \quad \text{or} \quad c_{u,i} = 1 + \alpha \log(1 + r_{u,i} / \epsilon),
\end{equation}
where $\alpha$ and $\epsilon$ are tunable hyperparameters. Note that the transform $c_{u,i}$ ensures that negative instances have small but non-zero weight, whereas positive instances receive increasingly higher weight according to their associated confidence.

%Paper 1: Collaborative Filtering for Implicit Feedback Datasets

\cite{pan2008one} approach the problem in a similar way, also fitting a function of the form in \Cref{eq:weightedMF}, though their weighting scheme is applied to negative instances. Several schemes are proposed, two of which are as follows:
\begin{equation}
  c_{u,i} = \alpha \times | I_u |; \quad \text{or} \quad c_{u,i} = \alpha ( m - | U_i | ).
\end{equation}
The first (which they call `user oriented' weighting) suggests that a negative instance should be weighted higher if the corresponding \emph{user} has interacted with many items; the second assumes that a negative instance should be weighted higher if the corresponding \emph{item} has few associated interactions. 
%\cite{pan2008one} also explores sampling schemes,



% Although the scheme above is ultimately a simple heuristic for reweighting the model we developed in \Cref{eq:MFObjective}, experiments in \cite{hu2008collaborative} show that this scheme outperforms models that try to predict $r_{u,i}$ or $p_{u,i}$ directly.

Although the schemes above are ultimately simple heuristics for reweighting the model we developed in \Cref{eq:MFObjective}, experiments in \cite{hu2008collaborative}
and
\cite{pan2008one}
show that these scheme outperforms models that try to predict $p_{u,i}$ (or $r_{u,i}$) directly.


%\subsection{Contrastive Learning for Personalized Ranking}
%\todo{generic section title, could be better}
\subsection{Bayesian Personalized Ranking}

While the above reweighting schemes demonstrate the importance of treating `negative' and `positive' feedback carefully in implicit-feedback settings, 
%they are ultimately simple heuristics whose success depends upon whether they can be adapted or tuned to a specific application or dataset.
they ultimately optimize \emph{regression} objectives, and therefore still seek to assign `negative' scores to unseen instances.

A potential objection to such an approach is that the unseen instances are exactly the ones we \emph{want} to recommend, and thus we should not encourage a model to assign them a negative score. A weaker assumption might state that while unseen instances should have \emph{lower} scores than positive instances, they need not have negative scores. That is, items which we know a user likes are more positive than unseen items, but unseen items could still be positive.

%Paper 2: BPR
%The above papers treated positive and negative instances differently, but still relied on regression objectives. As such, non-observed instances are still 
\cite{rendle2012bpr} built models based on the above principle by borrowing ideas from ranking. Recall from \Cref{sec:ranking} that the above principle is similar to our goal when adapting classifiers for ranking: while positive (or relevant) items should appear near the top of a ranked list, we are not concerned with their actual scores.

Likewise, the principle behind their method, Bayesian Personalized Ranking, is that we should generate ranked lists of items such that positive items appear first. This is achieved by training a predictor $x_{u,i,j}$ that assigns a score based on which of the two items ($i$ or $j$) is preferred (i.e.,~ranked higher) by $u$:
\begin{eqnarray}
    x_{u,i,j} > 0 & \rightarrow & \text{$u$ prefers $i$}\\
    x_{u,i,j} \leq 0 & \rightarrow & \text{$u$ prefers $j$}.
\end{eqnarray}
% \begin{equation}
%     \underbrace{\texttt{+++++++++}}_{\text{positives}}\underbrace{\texttt{?????????????}}_{\text{unseen items}}
% \end{equation}
Now, if we know that $i$ is a positive and $j$ is a negative (or unseen) item for user $u$, then a good model should tend to output positive values of $x_{u,i,j}$.
%we can measure whether the model correctly assigns a positive score by $\delta(x_{u,i,j} > 0)$, which we will later use to define our objective function.
%\todo{poor organization}

$x_{u,i,j}$ could be any predictor, though the most straightforward option is to define it in terms of \emph{difference} between pairwise predictors, e.g.:
\begin{equation}
    x_{u,i,j} = \underbrace{x_{u,i}}_{\mathllap{\text{$u$'s preference toward $i$}}\hspace{-5mm}} - \underbrace{x_{u,j}}_{\hspace{-5mm}\mathrlap{\text{$u$'s preference toward $i$}}}.
\end{equation}
For instance, the pairwise compatibility could be defined via a latent factor model, similar to that of \Cref{eq:???}:
\begin{equation}
    x_{u,i,j} = x_{u,i} - x_{u,j} = \gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j.
\end{equation}
%\todo{Replace all \clap with \mathclap or silimar (same for rlap and llap), which preserves font size}
Again, note that our goal is not that $\gamma_u \cdot \gamma_i$ should be positive for the positive item, nor thot $\gamma_u \cdot \gamma_j$ should be negative for the unseen item, only that the \emph{difference} is positive, i.e.,~that the positive item has a higher compatibility score.

We can now define our objective in terms of whether the model correctly outputs positive values $x_{u,i,j}$ given a positive item $i$ and an unseen item $j$.
%\todo{Kind of want to use x_{uij} rather than x_{u,i,j} (etc.) everywhere}
Ideally, we would like to count how often the model is able to correctly rank positive items higher than unseen items. For a specific user $u$ we have:
\begin{equation}
    \mathrm{AUC}(u) = \frac{1}{|I_u||\mathcal I \setminus I_u|} \underbrace{\sum_{i \in I_u \vphantom{j \in \mathcal I \setminus I_u}}}_{\mathllap{\text{positive items for user $u$}}\hspace{-5mm}} \underbrace{\sum_{j \in \mathcal I \setminus I_u}}_{\hspace{-5mm}\mathrlap{\text{unseen items for user $u$}}} \delta(x_{u,i,j} > 0).
    \label{eq:bprOpt1}
\end{equation}
The name `AUC' stands for `Area Under the ROC Curve;' this measure is equivalent to computing the area under ROC curve as we introduced in \Cref{sec:?}.
%which originates from a different way of deriving the same quantity.\footnote{Specifically, it is the Area Under the Receiver-Operating Characteristic Curve, which is a curve that relates the false positive and true positive rates of a classifier.}
%\todo{Cover ROC curve along with Precision Recall curves, and drop this footnote and previous sentence.}
For an entire dataset we average the above across all users:
\begin{equation}
    \mathrm{AUC} = \frac{1}{| \mathcal U |} \mathrm{AUC}(u)
\end{equation}
%\todo{AUC should probably have an argument, like AUC(\gamma|something)}
Note that this quantity takes a value between 0 and 1, where an AUC of 1 means that the model always ranks positive items higher than unseen items; an AUC of 0.5 means that the model is no better than random.

Optimizing the above presents two issues. First, it is not feasible to consider all $(u,i,j)$ triples; to address this one can randomly sample a fixed number of unseen items $j$ per positive item $i$.\footnote{Furthermore, the sampled items could change during each iteration of training.}

Second, the objective in \Cref{eq:bprOpt1} is a step function, whose derivative is zero almost everywhere. This is much the same issue we encountered when developing logistic regression in \Cref{sec:logisticRegression}; as such we can take the same approach by replacing the step function $\delta(x_{u,i,j})$ with a differentiable surrogate (see \Cref{fig:sigmoid}). Using the sigmoid function allows us to interpret $\sigma(x_{u,i,j})$ as a probability:
\begin{equation}
    p(\text{$u$ prefers $i$ over $j$}) = \sigma(x_{u,i,j}).
\end{equation}
From this point, optimization proceeds in much the same way as we developed logistic regression: we use $\sigma(x_{u,i,j})$ to define a (log-)probability of a model given a training set, and subtract a regularizer:
\begin{eqnarray}
  \mathrm{obj}^{\text{(BPR)}} & = & \ell (\gamma ; \mathcal T) - \lambda \Omega(\gamma)\\
                  & = & \sum_{(u,i,j) \in \mathcal T} \ln \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j) - \lambda \Omega(\gamma).
\end{eqnarray}
Assuming an $\ell_2$ regularizer $\Omega(\gamma) = \| \gamma \|_2^2$, we can compute the derivative, for example with respect to $\gamma_{u,k}$:
\begin{eqnarray}
    \frac{\partial \text{obj}}{\partial \gamma_{u,k}} & = & \sum_{(u,i,j) \in \mathcal T} \ln \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j) - \lambda \| \gamma \|_2^2\\
    & = & \frac{\partial}{\partial \gamma_{u,k}} \sum_{(u,i,j) \in \mathcal T} -\ln(1 + e^{\gamma_u \cdot \gamma_j - \gamma_u \cdot \gamma_i}) - \lambda \| \gamma \|_2^2\\
    & = & \sum_{(i,j) \in I_u} (\gamma_{i,k} - \gamma_{j,k}) (1 - \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j)) - 2\lambda\gamma_{u,k}
\end{eqnarray}
%\todo{Use \ln and \log consistently}
%\todo{\mathcal l or \ell ?}
%\todo{\gamma | T , or \gamma ; T, or something else?}
%\todo{Do I ever call the regularizer an \ell_2 regularizer? Maybe just drop}
(we abuse notation slightly so that $(i,j) \in I_u$ includes both positive and unseen items sampled from a user).
Derivatives of other terms can be computed similarly. In practice, it is often preferable to use libraries that compute such derivatives automatically, as we explore in \Cref{sec:recTensorflow}.

\section{Evaluating Recommender Systems}

%see e.g. http://ceur-ws.org/Vol-910/rue2012.pdf

So far, when developing models to predict ratings as in \Cref{sec:latentFactor}, we have done so by optimizing objectives based on a \emph{sum of squared errors} (or equivalently, a Mean Squared Error). Recall that in \Cref{sec:whyMSE}, we discussed the motivation behind the Mean Squared Error, as well as some potential pitfalls when using it.

In the case of recommender systems, we must be aware of the same pitfalls, but also some different ones. Critically, since since the system is likely used to provide ranked lists of items to the user, actual prediction of ratings may not be critical so long as desirable items appear near the top of the ranking.  
%
% Paper: Precision-Oriented Evaluation of Recommender Systems:
% An Algorithmic Comparison
%
%misclassified decoys (see refs in http://proceedings.mlr.press/v81/ekstrand18b/ekstrand18b.pdf)
%
Consider, for example, some potential problems with the Mean Squared Error in a recommendation context:
\begin{itemize}
  \item When using the MSE, mispredicting a 5-star rating as 4-stars incurs a smaller penalty than mispredicting a 3-star rating as 1-star. Arguably, the latter should have a smaller penalty, as it concerns an item which should never have been recommended anyway.
  \item Similarly, mispredicting ratings of 3 and 3.5 as 4 and 4.5 (respectively) would incur a larger penalty than mispredicting them as 3.5 and 3. However the former preserves the \emph{ordering} of the two items, whereas the latter does not.
  \item As we saw in \Cref{sec:motivationBehindMSE}, the MSE corresponds to an implicit assumption that errors are normally distributed; critically this assumes that outliers are extremely rare and should be penalized accordingly. In practice, outliers may be common, or alternately errors could be bimodal (or otherwise violate our model assumptions).
  \item \cite{bellogin2011precision} noted the issue of `popularity bias,' whereby strong performance on popular items can mask performance issues for less-popular ones.\footnote{Though this is not an issue with the MSE specifically, but rather a general problem of evaluation in imbalanced datasets.}
\end{itemize}

Ultimately, such problems raise the question of whether reductions in Mean Squared Error actually correspond to increased utility of a recommender system. Interestingly, it is reported in \cite{koren2009collaborative} that a carefully implemented model with temporally-evolving bias terms (which we describe in \Cref{sec:caseStudyNetflixTemporal}) outperformed Netflix's previous solution (\emph{Cinematch}) in terms of the RMSE. Critically, a system without any interaction term can do little more than recommend popular items over time; 

Some of the above issues suggest the use of alternative regression metrics, such as the Mean Absolute Error, which (for example) is less sensitive to outliers, as we argued in \Cref{sec:where?}. 
%Others require more subtle solutions; for example
Others suggest that perhaps a recommender system should be evaluated less like a regression problem and more like a ranking problem. That is, so long as items matching a user's interests have the \emph{highest} predicted scores, the precise accuracy of our predictions is unimportant.

Arguably, such problems with the Mean Squared Error are driving research toward settings that rely on implicit feedback (clicks, purchases, etc.)~rather than ratings; alternately, these settings may be preferable simply because such data is more available than explicit feedback such as ratings.
More crudely, optimizing clicks or purchases may simply correspond more closely to business goals compared to identifying highly-rated items.

We already saw in \Cref{sec:BPR} one technique to train recommender systems to optimize a ranking loss based on implicit feedback, namely the AUC. Conceptually, the AUC reflects our ability to guess which of two items is `relevant:' an AUC of 1 means that we always correctly select the correct item, whereas an AUC of 0.5 means our guesses are no better than random.

However, the AUC is but one choice of ranking loss, and was primarily chosen for its convenience when formulating the optimization problem in \Cref{eq:BPRloss?}. As in \Cref{sec:rankingLosses?}, when considering cases where recommendations are surfaced to a user via an interface, we may be particularly interested in how the recommender system performs among the top $K$ ranked items.

Below we present a few alternative evaluation functions to measure recommendation performance, most of which are focused on achieving high accuracy among the top-ranked items.

\paragraph{Recall @ $K$ and Precision @ $K$}

When we evaluated classifiers in \Cref{sec:evaluatingClassifiers}, we motivated the precision and recall @ $K$ useful metrics in the context of evaluating user interfaces, where we have a fixed budget ($K$) of results that can be returned. Likewise, when recommending items, we might consider whether relevant items (e.g.~those that a user eventually interacts with) are given a high ranking.

For convenience, when evaluating recommenders in this setting, it is useful to define a variable $\mathit{rank}_{u,i}$ that specifies in what position an item $i$ was ranked for a particular user $u$. That is, given a compatibility function $f(u,i)$, and a set of $N$ items that can potentially be recommended (potentially excluding e.g.~interactions that already appeared in the training set), then $\mathit{rank}_{u,i} \in \lbrace 1 \ldots N \rbrace$ such that
\begin{eqnarray}
    \mathit{rank}_{u,i} < \mathit{rank}_{u,j} & \Leftrightarrow & f(u,i) > f(u,j)\\
    \mathit{rank}_{u,i} = \mathit{rank}_{u,j} & \Leftrightarrow & i = j.
\end{eqnarray}
Now, given a test set of observed interactions $I_u$ we could define the precision@$K$
%\todo_{formatting}
(for a particular user $u$) as
\begin{equation}
  \mathit{prec}@K(u) = \frac{|\lbrace i \in I_u | \mathit{rank}_{u,i} \leq K \rbrace|}{K}.
\end{equation}
As in \Cref{eq:?}, the numerator is the number of relevant items that were retrieved, while the denominator is the number of retrieved items. Now to compute the precision @ $K$ we simply average over all users:
\begin{equation}
    \mathit{prec}@K = \frac{1}{|U|} \sum_{u \in U}\mathit{prec}@K(u).
\end{equation}
%\todo_{formatting, rank_{u,i} or rank(u,i), also prec or precision or Precision or \text{Precision}}
Likewise the recall @ $K$ is defined similarly:
\begin{equation}
    \mathit{recall}@K = \frac{1}{|U|} \sum_{u \in U} \frac{|\lbrace i \in I_u | \mathit{rank}_{u,i} \leq K \rbrace|}{|I_u|}
\end{equation}

%\paragraph{Hit Rate}

%\Done{Are there any papers that use recall or do they all call it hit rate? Addressed in footnote above}

%One potential challenge with the precision and recall as defined above is that we are averaging across users who may have very different numbers of relevant items $I_u$ in the test set; optimizing an algorithm 
%\done{Don't know what I had in mind here}

\paragraph{Mean Reciprocal Rank}
%\done{All paragraphs need . in current format. No they don't}
%
%\todo_{Above is quite sloppy}
%
%Mean Average Precision
%
The \emph{Mean Reciprocal Rank} (MRR) is another metric to assess whether a recommender system (or any classifier) ranks positive items highly; unlike the precision and recall @ $K$
%\todo{}
this expression does not depend on a particular size of the returned set of items, but rather rewards methods for ranking relevant items near the top of the list.
%\todo{messy description}

Traditionally, in search settings, the Mean Reciprocal Rank is defined in terms of the \emph{first} relevant item among a ranked list of retrieved results, though in recommendation settings the metric is typically used by building a test set that consists of only a single relevant item per user, $i_u$. Then, the Mean Reciprocal Rank is defined in terms of the inverse (reciprocal) of the rank of the relevant item:
\begin{equation}
    \mathit{MRR} = \frac{1}{|U|} \sum_{u \in U} \frac{1}{\mathit{rank}_{u,i_u}}.
\end{equation}
This metric will typically rank between 1 (if the relevant item is always ranked in the first position) and 0.5 (if the relevant item appears near the middle, or at a random position in the ranked list).
%\todo{written when drunk}

\paragraph{Cumulative Gain and NDCG}
%\todo{Not great paragraph title}
The \emph{Cumulative Gain} (and its variants) aim to measure ranking performance in a setting that resembles a user browsing a page of search results: relevant results should be among the top $K$ results, and ideally should be close to the top of the ranked list.
%
The \emph{Cumulative Gain} simply counts the number of relevant items among the top $K$ results:
\begin{equation}
    \text{Cumulative Gain}@K = \sum_{i \in \lbrace i | \mathit{rank}_{u,i_u} \leq K \rbrace} y_{u,i},
\end{equation}
where $y_{u,i}$ is either a binary label (e.g.~whether an item was purchase) or a relevance score (such as a rating). That is, the Cumulative Gain will be high if there are many relevant items (or highly rated items) among the top $K$ results.

Ideally, relevant results should appear closer to the top of the list; the \emph{Discounted} Cumulative Gain (DCG) accomplishes this by discounting the reward for items in lower ranks: 
%(so that an ideal ranker would rank all items in decreasing order of relevance):
\begin{equation}
    \text{DCG}@K = \sum_u \sum_{i \in \lbrace i | \mathit{rank}_{u,i_u} \leq K \rbrace} \frac{y_{u,i}}{\log_2(\mathit{rank}_{u,i} + 1)}.
    \label{eq:DCG}
\end{equation}
This expression is often normalized by comparison against an idealized ranking function to obtain the \emph{Normalized} Discounted Cumulative Gain (NDCG):
\begin{equation}
    \text{NDCG}@K = \frac{\mathit{DCG}@K}{\mathit{IDCG}@K}
\end{equation}
where $\mathit{IDCG}@K$ is the `ideal' discounted cumulative gain, i.e.,~the discounted cumulative gain that would have been achieved via an optimal ranking function $\mathit{rank}^{\text{opt}}_{u,i}$ (i.e.,~where labels $y_{u,i}$ are sorted in decreasing order of relevance). This normalization, and the specific choice of logarithmic scaling in \Cref{eq:DCG} are theoretically justified in
\cite{wang2013theoretical}.



\paragraph{Novelty, Diversity, Serendipity, etc.}
%\todo{Should I mention all of those things? I don't think they're covered elsewhere}

Serendipity: \cite{herlocker2004evaluating} 
%(serendipity, according to http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.62.9683&rep=rep1&type=pdf)

Coverage: An algorithmic framework for performing collaborative filtering (maybe)?
%according to same ref above

Finally, it should be noted that there are several other qualities we might desire from a recommender system beyond its immediate utility to users (i.e.,~its ability to predict the next action). For example, we might be interested in exposing users to diverse viewpoints, ensuring that recommendations aren't biased against certain groups, that users don't get pushed toward `extreme' content, etc. We'll revisit these issues in \Cref{chap:ethics}, when we consider the broader consequences and ethics of personalized machine learning.
%\todo{check above paragraph}


% \subsection{User Satisfaction}

% \subsection{Online Evaluation of Recommender Systems}

\subsection{Online Evaluation}

See some refs in http://proceedings.mlr.press/v81/ekstrand18b/ekstrand18b.pdf

\section{Other Aggregation Functions and Deep-learning Based Recommendation}
%\section{Aggregation Functions for Recommender Systems}
%\label{sec:aggregation}
%\todo{funny section titles here, just replace with "deep learning" or something}

\subsection{Why the Inner Product?}
%\todo{drop?}

Briefly, it is worth mentioning that the type of recommendation objective in \Cref{sec:?,sec:?} in which we measure compatibility via an inner product is only one potential choice of compatibility relationship.

While measuring compatibility between a user $u$ and item $i$ in terms of $\gamma_u \cdot \gamma_i$ arises naturally when deriving solutions in terms of matrix factorization

\emph{Neural Collaborative Filtering} \cite{he2017neural} argued that deep learning approaches can be used to learn 
%\todo{confirm this is actually what the paper says}

Discuss that recent paper by the BPR author
\cite{rendle2020neural}



Could discuss deep learning papers: e.g. the two youtube papers

%\todo{Rename section probably}

\section{Deep Learning for Recommendation}

Some from
\cite{zhang2019deep}

\cite{zhang2019deep} discuss various potential benefits and limitations of deep learning-based recommender systems. Arguably, the main benefit of deep learning-based is the ability to uncover complex, non-linear relationships between user and item representations. For example, previously (\Cref{sec:?})
%\todo{was it previously?}
we discussed the relative benefits of the inner-product versus a Euclidean metric when comparing user and item representations. In principle, deep learning approaches could learn more flexible aggregation functions aggregation functions, reducing the need for manual engineering.

\cite{zhang2019deep} also point other appealing properties of deep learning-based recommenders, including the effectiveness of deep learning when dealing with structured data such as sequences, images, or text, and the ubiquity of high-level libraries that facilitate straightforward implementation. 
%\todo{Depending on organization, may have already mentioned libraries}
We discuss such 

%\begin{description}
\subsection{Multilayer Perceptron-based Recommendation}
%\todo{Could move to "basics" chapter, but maybe not}

%https://github.com/davidstutz/latex-resources/blob/master/tikz-multilayer-perceptron/multilayer-perceptron.tex
\begin{figure}[t]
	\centering
	\begin{tikzpicture}[shorten >=1pt]
		\tikzstyle{unit}=[draw,shape=circle,minimum size=0.8cm]
% 		\tikzstyle{hidden}=[draw,shape=circle,fill=black!25,minimum size=1.15cm]
		\tikzstyle{hidden}=[draw,shape=circle,minimum size=0.8cm]

		\node[unit](x0) at (0,3.5){$x_0$};
		\node[unit](x1) at (0,2){$x_1$};
		\node at (0,1){\vdots};
		\node[unit](xd) at (0,0){$x_F$};

		\node[hidden](h10) at (2,4){$h_0^{(1)}$};
		\node[hidden](h11) at (2,2.5){$h_1^{(1)}$};
		\node at (2,1.5){\vdots};
		\node[hidden](h1m) at (2,-0.5){$h_{m}^{(1)}$};

		\node(h22) at (3.5,0){};
		\node(h21) at (3.5,2){};
		\node(h20) at (3.5,3.75){};
		
		\node(d3) at (4,0){$\ldots$};
		\node(d2) at (4,2){$\ldots$};
		\node(d1) at (4,3.75){$\ldots$};

		\node(hL12) at (4.5,0){};
		\node(hL11) at (4.5,2){};
		\node(hL10) at (4.5,3.75){};
		
		\node[hidden](hL0) at (6,4){$h_0^{(L)}$};
		\node[hidden](hL1) at (6,2.5){$h_1^{(L)}$};
		\node at (6,1.5){\vdots};
		\node[hidden](hLm) at (6,-0.5){$h_{m}^{(L)}$};

% 		\node[unit](y1) at (12,3.5){$y_1^{(L+1)}$};
		\node[unit](y2) at (8,1.75){$y$};
% 		\node at (12,1){\vdots};	
% 		\node[unit](yc) at (12,0){$y_C^{(L+1)}$};

        \draw[->] (x0) -- (h10);
		\draw[->] (x0) -- (h11);
		\draw[->] (x0) -- (h1m);

        \draw[->] (x1) -- (h10);
		\draw[->] (x1) -- (h11);
		\draw[->] (x1) -- (h1m);

        \draw[->] (xd) -- (h10);
		\draw[->] (xd) -- (h11);
		\draw[->] (xd) -- (h1m);

% 		\draw[->] (hL0) -- (y1);
% 		\draw[->] (hL0) -- (yc);
		\draw[->] (hL0) -- (y2);

% 		\draw[->] (hL1) -- (y1);
% 		\draw[->] (hL1) -- (yc);
		\draw[->] (hL1) -- (y2);

% 		\draw[->] (hLm) -- (y1);
		\draw[->] (hLm) -- (y2);
% 		\draw[->] (hLm) -- (yc);

        \draw[->,path fading=east] (h10) -- (h20);
		\draw[->,path fading=east] (h10) -- (h21);
		\draw[->,path fading=east] (h10) -- (h22);
		
		\draw[->,path fading=east] (h11) -- (h20);
		\draw[->,path fading=east] (h11) -- (h21);
		\draw[->,path fading=east] (h11) -- (h22);
		
		\draw[->,path fading=east] (h1m) -- (h20);
		\draw[->,path fading=east] (h1m) -- (h21);
		\draw[->,path fading=east] (h1m) -- (h22);
		
		\draw[->,path fading=west] (hL10) -- (hL0);
		\draw[->,path fading=west] (hL11) -- (hL0);
		\draw[->,path fading=west] (hL12) -- (hL0);
		
		\draw[->,path fading=west] (hL10) -- (hL1);
		\draw[->,path fading=west] (hL11) -- (hL1);
		\draw[->,path fading=west] (hL12) -- (hL1);
		
		\draw[->,path fading=west] (hL10) -- (hLm);
		\draw[->,path fading=west] (hL11) -- (hLm);
		\draw[->,path fading=west] (hL12) -- (hLm);
		
		\draw [yshift=0pt] (x0) node [yshift=+0.7cm]{input layer};
		\draw [yshift=0pt] (h10) node [yshift=+0.7cm]{hidden layer};
		\draw [yshift=0pt] (hL0) node [yshift=+0.7cm]{hidden layer};
		\draw [yshift=0pt] (y2) node [yshift=+0.7cm]{output};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (2.5,4.5) -- (3.75,4.5) node [black,midway,yshift=+0.6cm]{$1^{\text{st}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (8.5,4.5) -- (9.75,4.5) node [black,midway,yshift=+0.6cm]{$L^{\text{th}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (11.5,4) -- (12.75,4) node [black,midway,yshift=+0.6cm]{output layer};
	\end{tikzpicture}
	\caption{Representation of a multi-layer perceptron with $L$ layers. Here for simplicity each layer has the same number ($m$) of units, and we have only a single output $y$ (so that the inputs and outputs are similar to the regression and classification problems we studied in \Cref{chap:basics}).}
	\label{fig:multilayer-perceptron}
\end{figure}
%\todo{Write short captions for all figures}


Multilayer perceptrons (MLPs) are a staple of artificial neural networks, offering a straightforward way to learn non-linear transformations and interactions among features.

Roughly speaking, a `layer' of a multilayer perceptron transforms a vector of input variables to a (possibly lower dimensional) vector of output variables; 
%via a linear function of the input 
typically the output variables are related to the input variables via a linear transformation followed by a non-linear activation, e.g.:
\begin{equation}
    f(x) = \sigma(Mx).
\end{equation}
%where $\sigma$ is applied elementwise.
Here $x$ is a vector of input variables, $f(x)$ is a vector of output variables, and $M$ is a learned matrix, such that each term in $Mx$ is a weighted combination of the original features in $x$. The sigmoid function is applied elementwise, in this case transforming the output variables to lie in the range $(0,1)$.

While the above is just one \emph{layer} of a multilayer perceptron, several such layers can be `stacked' in order for the network to learn complex non-linear functions.
%\todone{(nothing else to do?)}
%
Eventually, the final layer predicts some desired output, e.g.~a regression or classification objective. For example, the final layer might simply take a weighted combination of features from the previous layer:
\begin{equation}
    f(x) = \sigma(\theta \cdot x),
\end{equation}
i.e.,~similar to the output of a logistic regressor (for a classification task).

We depict a multilayer perceptron in \Cref{fig:multilayer-perceptron}. Note that a trivial linear model of the form $y = \theta x$
%\todo{notation}
would be depicted by a similar figure in which the inputs were connected directly to the output.

Ultimately, multilayer perceptrons handle similar modalities of data and problems to those we saw in \Cref{chap:basics}, i.e.,~feature vectors as inputs and regression or classification targets as outputs; the main difference compared to the models from \Cref{chap:basics} is simply in their ability to learn complex non-linear transformations and relations among features.
%\todo{multilayer or multi-layer}

\paragraph{Neural Collaborative Filtering}
    %\todone{I think}
    %Approaches include neural collaborative filtering
\cite{he2017neural} attempted to apply the benefits of multilayer perceptrons to latent-factor recommender systems. The essential idea is fairly simple: rather than combining user and item latent factors via an inner product (as in \Cref{eq:?}), $\gamma_u$ and $\gamma_i$ are combined via a multilayer perceptron to predict the model output (note that both the latent factors and the MLP parameters are learned simultaneously). As we discussed in \Cref{sec:?}
%\todo{check that this exists and comes before this discussion}
the inner product function is only one possible choice when combining user and item preferences and other choices (such as a Euclidean distance) may be more appropriate in other settings; conceptually, the promise of a solution based on a multilayer perceptron is that one can be agnostic to these choices with the expectation that the model will learn the correct aggregation function automatically. While \cite{he2017neural} showed this method to be effective in some settings, there has recently been some question as to the value of this type of technique: while MLPs can in principle learn quite general functions, in practice specific functions (like the inner product) are not easily recoverable by such models, meaning that simpler models may still often outperform these more complex approaches. We discuss this issue further in \Cref{sec:howEffective}.
%\todo{include figure for architecture?}
%\todo{also include deepFM figure, see https://d2l.ai/chapter_recommender-systems/deepfm.html}

\paragraph{Deep Factorization Machines}
%and deep factorization machines 
%nice figure: https://d2l.ai/chapter_recommender-systems/deepfm.html
%
Likewise, \cite{guo2017deepfm} attempted to use multilayer perceptrons to improve the performance of factorization machines. The idea is similar to what we presented above: just as user and item embeddings were combined above, \cite{guo2017deepfm} combine the embeddings of several terms (users, items, previous items, etc.) via an MLP. In practice, this MLP is trained in parallel to a traditional factorization machine, and the outputs are combined in 
%\todo{doesn't make sense since we haven't discussed FMs yet! Move to next section}

\cite{he2017neuralB}


    
\subsection{Autoencoder-based Recommendation}

Roughly speaking, the role of an \emph{autoencoder} is to learn a low-dimensional representation of some input data that preserves the ability to reconstruct the original (high-dimensional) data from the low-dimensional representation.
%\todo{cite some examples of autoencoders}
%\cite{zhang2019deep} suggest 
%to 
%\todo{This sentence}
%common ways in which this principle can be applied in the context of recommendation: first, 
%AutoRec: \cite{sedhain2015autorec}
The basic principle of an autoencoder is depicted in \Cref{fig:autoencoder}. Here an input vector $x$ is projected into a lower-dimensional space via a function $g(x)$ (following an approach similar to that followed by a multi-layer perceptron above, which may include several layers). The low-dimensional representation is then mapped back into the original space via $f(g(x))$; the goal is that $f(g(x))$ should match the original data $x$ as closely as possible. In this way $g(x)$ acts as a `bottleneck,' forcing the model to learn a compressed representation that succinctly captures the meaningful information in $x$. Several variants of autoencoders exist, for instance \emph{denoising autoencoders} partially corrupt the input in order to learn representations that are robust to noise; \emph{sparse autoencoders} attempt to learn compressed representations that are sparse, etc.

\begin{SCfigure}
	\centering
	\begin{tikzpicture}[shorten >=1pt]
		\tikzstyle{unit}=[draw,shape=circle,minimum size=0.8cm]
% 		\tikzstyle{hidden}=[draw,shape=circle,fill=black!25,minimum size=1.15cm]
		\tikzstyle{hidden}=[draw,shape=circle,minimum size=0.8cm]

		\node[hidden](h10) at (2,4){$x_0$};
		\node[hidden](h11) at (2,2.5){$x_1$};
		\node at (2,1.25){\vdots};
		%\todo{Fix position of vdots in other figure too}
		\node[hidden](h1m) at (2,-0.5){$x_n$};

% 		\node(h22) at (3.5,0){};
% 		\node(h21) at (3.5,2){};
% 		\node(h20) at (3.5,3.75){};
		
		\node[hidden](d3) at (4,0){$g_0$};
		\node[hidden](d2) at (4,2){$g_1$};
		\node[hidden](d1) at (4,3.75){$g_k$};
		%\todo{Why don't the arrows go all the way to the thing? Copy code from other figure to fix}

% 		\node(hL12) at (4.5,0){};
% 		\node(hL11) at (4.5,2){};
% 		\node(hL10) at (4.5,3.75){};
		
		\node[hidden](hL0) at (6,4){$f_0$};
		\node[hidden](hL1) at (6,2.5){$f_1$};
		\node at (6,1.25){\vdots};
		\node[hidden](hLm) at (6,-0.5){$f_n$};

% 		\node[unit](y1) at (12,3.5){$y_1^{(L+1)}$};
% 		\node[unit](y2) at (8,1.75){$y$};
% 		\node at (12,1){\vdots};	
% 		\node[unit](yc) at (12,0){$y_C^{(L+1)}$};

% 		\draw[->] (hL0) -- (y1);
% 		\draw[->] (hL0) -- (yc);
% 		\draw[->] (hL0) -- (y2);

% 		\draw[->] (hL1) -- (y1);
% 		\draw[->] (hL1) -- (yc);
% 		\draw[->] (hL1) -- (y2);

% 		\draw[->] (hLm) -- (y1);
% 		\draw[->] (hLm) -- (y2);
% 		\draw[->] (hLm) -- (yc);

        \draw[->] (h10) -- (d1);
		\draw[->] (h10) -- (d2);
		\draw[->] (h10) -- (d3);
		
		\draw[->] (h11) -- (d1);
		\draw[->] (h11) -- (d2);
		\draw[->] (h11) -- (d3);
		
		\draw[->] (h1m) -- (d1);
		\draw[->] (h1m) -- (d2);
		\draw[->] (h1m) -- (d3);
		
		\draw[->] (d1) -- (hL0);
		\draw[->] (d2) -- (hL0);
		\draw[->] (d3) -- (hL0);
		
		\draw[->] (d1) -- (hL1);
		\draw[->] (d2) -- (hL1);
		\draw[->] (d3) -- (hL1);
		
		\draw[->] (d1) -- (hLm);
		\draw[->] (d2) -- (hLm);
		\draw[->] (d3) -- (hLm);
		
		\draw [yshift=0pt] (h10) node [yshift=+0.7cm]{input};
		\draw [yshift=0pt] (d1) node [yshift=+0.7cm]{encoded};
		\draw [yshift=0pt] (hL0) node [yshift=+0.7cm]{decoded};
% 		\draw [yshift=0pt] (y2) node [yshift=+0.7cm]{output};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (2.5,4.5) -- (3.75,4.5) node [black,midway,yshift=+0.6cm]{$1^{\text{st}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (8.5,4.5) -- (9.75,4.5) node [black,midway,yshift=+0.6cm]{$L^{\text{th}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (11.5,4) -- (12.75,4) node [black,midway,yshift=+0.6cm]{output layer};
	\end{tikzpicture}
	\caption{Autoencoder. $g_i$ and $f_i$ are shorthand for $g(x)_i$ and $f(g(x))_i$ respectively. Either of the encoding or decoding operations could include multiple layers.}
	\label{fig:autoencoder}
\end{SCfigure}

\paragraph{AutoRec}
\cite{sedhain2015autorec} adapts the principle of autoencoders to recommendation problems. In their setting the data to be encoded is vector of item ratings for an item $i$, or equivalently a column of an interaction matrix $R_{\cdot,i}$. Since $R_{\cdot,i}$ is only partially observed (rather than a dense vector as in a traditional autoencoer), the compressed representation is only responsible for
(and gradient updates are only applied to)
the observed entries $R_{u,i}$. At inference time the compressed representation can then be used to estimate the entries for unobserved pairs. 
\cite{sedhain2015autorec} term this version of the model \emph{AutoRec-I}, since an autoencoder is used to learn compressed item representations; alternately, \emph{AutoRec-U} consists of the same approach applied to user vectors $R_{u,\cdot}$.

Note that AutoRec-I lacks any user parameters (and likewile AutoRec-U lacks item parameters). As such it is a form of \emph{user-free} personalization, that personalizes predictions using a model that aggregates data from the entire user history rather than `memorizing' user preferences via an explicit parameter. We further explore such notions of user-free personalization when exploring methods based on sequences and text in \Cref{sec:?}.
%\todo{make sure ordering is correct}

%\todo{Could mention other autoencoder methods, not critical}

\subsection{Convolutional and Recurrent Networks}

Finally, \cite{zhang2019deep} survey various recommender systems based on recurrent networks and convolutional neural networks. Recurrent networks are typically chosen as a way of exploring \emph{sequential} dynamics in user activities; we explore this type of approach in \Cref{chap:temporalSequential}, including deep learning based approaches in \Cref{?}. Convolutional neural networks are often used as a means of incorporating representations of rich content (such as images) into recommender systems; we explore this type of approach in \Cref{chap:visual}.
% \end{description}

\cite{zhang2019deep} 
%\todo{Not every paragraph should start with citation}
also discuss potential limitations of deep learning-based approaches, including the challenges involved in interpreting the predictions of deep learning systems, and the difficulty of tuning hyperparameters in systems with many complex, interacting parts. They also highlight the potential lack of interpretability of deep learning-based models (though this is to some extent a challenge with any model based on latent representations), as well as the `data hungriness' of deep learning approaches. The latter issue arises whenever fitting models with a large number of parameters, and indeed is also a problem when fitting traditional latent factor models as in \Cref{sec:?}. On the one hand, this may narrow the conditions under which deep learning-based approaches are effective, e.g.~they may underperform in cold-start situations where few interactions are available per user or per item. On the other hand, deep learning approaches may extend the modalities of data that can be incorporated into recommendation approaches (including in cold-start settings), for instance by leveraging text or image data; we explore such approaches in \Cref{??} and \Cref{?}.

%\subsection{Neural Collaborative Filtering and Deep Factorization Machines}
%\todo{Needs to move to next chapter after FMs are discussed}

%\subsection{Autoencoder Based Recommendation

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Deep learning-based recommendation techniques.}
\label{tab:dlRecExamps}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{lXX}
% \hrule\\
\textbf{Reference} & \textbf{Technique} & \textbf{Description}\\
\cite{he2017neural} & neural collaborative filtering & Uses multi-layer perceptrons to learn complex interactions between user and item latent factors.\\
\cite{guo2017deepfm} & deep factorization machines & Uses multi-layer perceptrons to train factorization machines. \\
\cite{he2017neuralB} & neural factorization machines & Trains factorization machines using the \emph{Wide \& Deep} approach from \cite{?}.\\
%\todo{say what the point of wide&deep is compared to MLP}
\cite{sedhain2015autorec} & AutoRec & Learns compressed representations of item (or user) interaction vectors; the compressed representations can be used to estimate scores associated with unobserved interactions.  \\
\end{tabularx}
\end{minipage}
\end{table}
%\todo{table formatting}

\subsection{How Effective are Deep Learning-Based Recommenders?}
\label{sec:howEffective}

In spite of the proliferation of deep learning-based recommendation approaches, their benefit over simpler, more traditional forms of recommendation is perhaps questionable.
\cite{dacrema2019we} conducted a thorough evaluation of several of the predominant deep learning-based recommeder systems (including several of the approaches discussed above), and found that deep learning approaches were often outperformed by simpler baselines, so long as those baselines were carefully tuned. Most of the models for recommendation we've seen so far involve many tunable factors (e.g.~number of factors, regularization schemes, and details of the specific training approaches), as well as choices in terms of dataset selection, pre-processing, etc.~that can favor certain models over others. 
%
Although the evaluation in \cite{dacrema2019we} was limited to a few specific (but popular) approaches, it raised broader issues of evaluation and benchmarking in recommender systems.
%\todo{could connect better to something actually in the above paper}
Some general points raised include the difficulty in reproducing reported results (while releasing research code is common practice, releasing exact hyperparameter settings or tuning strategies is not), and the proliferation of datasets, metrics, and evaluation protocols that make fair comparison difficult.

\cite{rendle2020neural} explored the same issue, focusing on the specific comparison of inner product-based recommendation versus solutions based on multilayer perceptrons.
%\todo{MLPs are defined somewhere above?}
They reiterate the main point from \cite{dacrema2019we} that simpler methods remain competitive so long as they're carefully tuned. They also argue that in spite of the hope that multilayer perceptrons can learn complex, non-linear relationships, that in practice even simple functions (like the inner product) are difficult for such models to reproduce.

Finally, both \cite{dacrema2019we} and \cite{rendle2020neural} 
discuss issues of computational complexity, and whether the marginal benefits of deep learning-based approaches justify the substantial added complexity.
\cite{rendle2020neural} 
argue that simpler models may be preferable in production environments, especially when considering the efficiency of item retrieval (as we discuss below).
%\todo{check whether it's still below}
%\todo{above paragraphs are fine but basically just the abstract from that paper}

Note that the above criticisms are not an indictment of deep learning-based approaches in general, but only with respect to their ability to model specific types of interaction data (essentially the same settings discussed in this chapter). In later chapters we'll explore the use of deep learning-based approaches in a variety of other settings, in order to model sequence, text, and image data, with goals ranging from cold-start performance to interpretability.

\section{Retrieval}
%\todo_{Probably collapse these to a paragraph somewhere}

Briefly, it is worth discussing one of the fundamental considerations when deploying a recommender system, namely, how can we efficiently retrieve items. Na\"ively, having defined a compatibility function $f(u,i)$
%\todo{is this the right notation?}
between a user and an item, our goal might be to rank (unseen) items according to their compatibility, i.e.,
\begin{equation}
    \mathit{rec}(u) = \argmax_{i \in \mathcal I \setminus I_u} f(u,i).
    %\todo{check notation}
    \label{eq:recsysRetrieve}
\end{equation}
Presumably, recommendations must be made rapidly, for use in interactive settings. Given a large vocabulary of items, this procedure is likely to be prohibitively expensive if we were to attempt to enumerate scores for all items $i\in \mathcal I$;
%\todo{notation}
as such it is worth thinking about what types of relevance functions $f(u,i)$ admit efficient solutions to \Cref{eq:recsysRetrieve}.

\paragraph{Euclidean Distance}
Perhaps the most straightforward function for efficient retrieval is a Euclidean distance function, as in \Cref{sec:aggregation?}, i.e.,
\begin{equation}
    f(u,i) = \| \gamma_u - \gamma_i \|.
\end{equation}
In this case, retrieval can be done efficiently (i.e.,~$O(\log(| \mathcal I |)$ on average) using traditional data structures such as a KD-tree. A KD-tree \cite{bentley1975multidimensional} is a data structure that represents $K$-dimensional points (in this case $\gamma_i$ for each item) in such a way as to allow efficient retrieval given a query ($\gamma_u$); such data structures predate recommender systems and have classical applications in nearest-neighbor retrieval for classification (for example).

%X-box inner product
%https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/XboxInnerProduct.pdf

\paragraph{Inner Product and Cosine Similarity}
\cite{bachrach2014speeding} showed that the same types of data structure can be adapted for other types of relevance function. Fundamentally, they showed that both inner product and cosine distance-based relevance functions can be related to nearest neighbor search (as above) via appropriate transformations:
\begin{eqnarray}
    \argmax_i \| \gamma_u - \gamma_i \| & \text{nearest neighbor (NN)}\\
    \argmax_i \gamma_u \cdot \gamma_i & \text{maximum inner product (MIP)}\\
    \argmax_i \frac{\gamma_u \cdot \gamma_i}{\| \gamma_u \| \| \gamma_i \|} & \text{maximum cosine similarity (MCS)}.
\end{eqnarray}
%\todo{Doesn't render properly}
%\todo{dunno if need abbreviations}
Doing so allows the same data structures that facilitate nearest-neighbor (Euclidean) search to be used for recommenders based on inner products (as in \Cref{sec:?}) or cosine similarity (as in \Cref{sec:?}).

\paragraph{Approximate Search and Jaccard Similarity}
%Other data structures can also be used, for example
In practice, approximation schemes may also be used for efficient retrieval, such as techniques based on locality-sensitive hashing (whereby `similar' items are hashed to the same bucket). Versions of locality-sensitive hashing have been proposed to retrieve similar items based on similarity functions including Euclidean distance \cite{indyk1998approximate}, 
Jaccard \cite{broder1997resemblance}, and
Cosine similarity \cite{charikar2002similarity}. 
\cite{bachrach2014speeding} compares exact techniques for retrieval (as described above) to these types of hashing-based approximations, as well as other exact techniques for recommendation \cite{koenigstein2012efficient}.

%Maximum inner product search

%Efficient nearest neighbor

%\subsection{Recommendation based on Euclidean Distance, Translation, Etc.}
%Covered in seequential recsys section



% \section{Recommending for Diversity}
%\todo{Could put in ethics chapter or whatever}

\section{Online Updates}
\label{sec:onlineUpdates}

Online-Updating Regularized Kernel Matrix Factorization
Models for Large-Scale Recommender Systems

\section{Recommender Systems in Python with \emph{Surprise} and \emph{Implicit}}

Although the types of recommender systems we've seen so far can (with some effort) be implemented `from scratch' either by computing the gradient expressions as in \Cref{eq:latentFactorGradient} or by using high-level optimization libraries like \emph{Tensorflow} (we'll explore a Tensorflow implementation in \Cref{?}),
%\todo{capitalization}
the recommendation techniques we've covered so far are reasonably well-supported by popular Python
%\todo{italics?}
libraries.

Here we examine two specific libraries, \emph{Surprise} and \emph{Implicit} for latent-factor recommendation (as in \Cref{sec:?}) and Bayesian Personalized Ranking (as in \Cref{sec:?}. However these examples serve more to introduce the overall recommendation pipeline, rather than to dive deeply into the specifics of these libraries.

\subsection{Latent-Factor Models (\emph{Surprise})}

\emph{Surprise} \cite{Hug2020}
%\todo{Modified citation, doesn't allow url}
is a library that implements various collaborative filtering
%\todo{Do I ever use this word?}
methods and algorithms based on \emph{explicit feedback} (e.g.~ratings). Below we show how to use \emph{Surprise}'s
%\todo{Formatting}
implementation of a latent factor model as in \Cref{eq:latentWhich?} (`SVD' as in \cite{Koren?}).

First we import the model (`SVD'), and utilities to read and split the dataset:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from surprise import SVD, Reader, Dataset
from surprise.model_selection import train_test_split
\end{lstlisting}
\end{minipage}

While the library has various routines to read in the dataset, the most straightforward is to read the data from a CSV/TSV file. Here we've processed the \emph{Goodreads} `fantasy' data
%\todo{Do I talk about this dataset above?}
to extract just the `user\_id,' `book\_id,' and `rating' fields, though this example could be applied to any similar dataset. After reading in the data, we split it into train and test fractions, with 25\% of the data withheld for testing:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
reader = Reader(line_format='user item rating', sep='\t')
data = Dataset.load_from_file("goodreads_fantasy.tsv", reader=reader)
dataTrain, dataTest = train_test_split(data, test_size=.25)
\end{lstlisting}
\end{minipage}

Next we fit the model, and collect its predictions on the test set:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
model = SVD()
model.fit(dataTrain)
predictions = model.test(dataTest)
\end{lstlisting}
\end{minipage}

From `predictions' we can then 
extract and compare the model's prediction (\texttt{p.est}) and the original value (\texttt{p.r\_ui}), in this case to compute the mean squared error:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
sse = 0
for p in predictions:
    sse += (p.r_ui - p.est)**2

MSE = sse / len(predictions)
\end{lstlisting}
\end{minipage}

%print(sse / len(predictions))

%predictions[0].est

\subsection{Bayesian Personalized Ranking (\emph{Implicit})}

\emph{Implicit}\footnote{\url{https://github.com/benfred/implicit}} is a library for recommender systems that operate on implicit feedback datasets, as in \Cref{sec:implicit}. Here we show how to use the library for Bayesian Personalized Ranking, 
%\todo{capitalization of BPR?}
as in \Cref{sec:bpr}.

First, we read in the data. This time, the required data format is a \emph{sparse} matrix describing all user/item interactions. Despite this matrix having hundreds of thousands of rows and columns, only observed interactions are stored:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from implicit import bpr

Xiu = scipy.sparse.lil_matrix((nItems, nUsers)) # Sparse matrix; initialized after extracting the number of users and items
for d in data:
    itemsToUsers[itemIDs[d['book_id']],userIDs[d['user_id']]] = 1 # Only storing positive feedback instances
    
Xui = scipy.sparse.csr_matrix(Xiu.T)
\end{lstlisting}
\end{minipage}
%\todo{Have I shown how to read the data? Could put here}
%\todo{Sometimes call it 'data' and sometimes 'dataset', should be consistent}

Next, we initialize and fit the BPR model:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
model = bpr.BayesianPersonalizedRanking(factors = 5)
model.fit(Xiu)
\end{lstlisting}
\end{minipage}
%\todo{Make the various examples have the same number of factors}

Having fit the model, we can retrieve the user and item factors $\gamma_u$ and $\gamma_i$, as well as recommendations (high $\gamma_u \cdot \gamma_i$) and similar items (high similarity to $\gamma_i$):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
itemFactors = model.item_factors
userFactors = model.user_factors

recommended = model.recommend(0, Xui) # Recommendations for the first user
related = model.similar_items(0) # Highly similar to the first item (cosine similarity)
\end{lstlisting}
\end{minipage}
%#

\subsection{Implementing a Latent Factor Model in Tensorflow}
%\todo{style}

Following our introduction to tensorflow
%\todo{style}
in \Cref{sec:?}, it is now fairly straightforward to implement the more complex models developed in this chapter. 
Here we fit a latent factor model following \Cref{sec:?}.
%\todo{slightly messy paragraph}

We start by initializing our model, which takes as parameters the model dimensionality $K$ and the regularization strength $\lambda$.
%We start by defining 
Here we define
our variables to be fit ($\alpha$, $\beta_u$, $\beta_i$, $\gamma_u$, $\gamma_i$). In practice, appropriate initialization of such variables is important; here alpha is initialized to the mean rating $\mu$ while all other parameters are initialized following a normal distribution:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
class LatentFactorModel(tf.keras.Model):
    def __init__(self, mu, K, lamb):
        super(LatentFactorModel, self).__init__()
        self.alpha = tf.Variable(mu)
        self.betaU = tf.Variable(tf.random.normal([len(userIDs)],stddev=0.001))
        self.betaI = tf.Variable(tf.random.normal([len(itemIDs)],stddev=0.001))
        self.gammaU = tf.Variable(tf.random.normal([len(userIDs),K],stddev=0.001))
        self.gammaI = tf.Variable(tf.random.normal([len(itemIDs),K],stddev=0.001))
        self.lamb = lamb
\end{lstlisting}
\end{minipage}
%\todo{Need to update my earlier TF examples to be consistent with this one?}

Next we define our function (a method in the same class) that makes a prediction for a given user/item pair, i.e.,~$f(u,i) = \alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i$ as in \Cref{eq:?}:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
    def predict(self, u, i):
        p = self.alpha + self.betaU[u] + self.betaI[i] +\
            tf.tensordot(self.gammaU[u], self.gammaI[i], 1)
        return p
\end{lstlisting}
\end{minipage}

Similarly we define our regularizer as in \Cref{sec:?} (which could easily be adapted to include different coefficients for different terms, for example):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
    def reg(self):
        return self.lamb * tf.reduce_sum(self.betaU**2) +\
                           tf.reduce_sum(self.betaI**2) +\
                           tf.reduce_sum(self.gammaU**2) +\
                           tf.reduce_sum(self.gammaI**2)
\end{lstlisting}
\end{minipage}

Finally we define the function to compute the squared error for a single sample, which will be called when computing gradients:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
    def call(self, u, i, r):
        return (self.predict(u,i) - r)**2
\end{lstlisting}
\end{minipage}
%
%model = LatentFactorModel(mu, 5, 0.00001)

%\todo{Shrink code, fix word wrapping}
%\todo{have code line numbers continue (everywhere)}


\subsection{Bayesian Personalized Ranking in Tensorflow}
%\todo{Change this and the above to be paragraphs rather than sections}

Bayesian Personalized Ranking (as in \Cref{sec:?}) can be implemented similarly. Again we initialize our model variables (this time only $\beta_i$, $\gamma_u$, and $\gamma_i$) are included:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
class BPR(tf.keras.Model):
    def __init__(self, K, lamb):
        super(BPR, self).__init__()
        self.betaI = tf.Variable(tf.random.normal([len(itemIDs)],stddev=0.001))
        self.gammaU = tf.Variable(tf.random.normal([len(userIDs),K],stddev=0.001))
        self.gammaI = tf.Variable(tf.random.normal([len(itemIDs),K],stddev=0.001))
        self.lamb = lamb
\end{lstlisting}
\end{minipage}
%\todo{shrink}

Our prediction function estimates the unnormalized score
$x_{u,i} = \beta_i + \gamma_u \cdot \gamma_i$:
%\todo{is this defined somewhere and is it called $x_{u,i}$ there?}
        
\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
    def predict(self, u, i):
        p = self.betaI[i] + tf.tensordot(self.gammaU[u], self.gammaI[i], 1)
        return p
\end{lstlisting}
\end{minipage}

% \noindent
% \begin{minipage}{\linewidth}
% \begin{lstlisting}[escapechar=\%]
%     def reg(self):
%         return self.lamb * tf.reduce_sum(self.betaI**2) +\
%                           tf.reduce_sum(self.gammaU**2) +\
%                           tf.reduce_sum(self.gammaI**2)
% \end{lstlisting}
% \end{minipage}

Finally we define our loss for a single sample, this time including a user $u$, and items $i$ and $j$ that they did and did not interact with:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
    def call(self, u, i, j):
        return -tf.math.log(tf.math.sigmoid(self.predict(u,i) - self.predict(u,j)))
\end{lstlisting}
\end{minipage}

%model = BPR(5, 0.00001)

\subsection{Efficient Batch-Based Optimization}

Although the above implementations are straightforward, they are not particularly efficient and are memory 
%\todo{not sure about memory, think about it}
hungry if we attempt to compute the complete MSE (\Cref{eq:?})
%\todo{style}
or likelihood (\Cref{eq:?}) across the entire dataset. Instead, we compute gradients and update parameters in \emph{batches} consisting of a random sample of our data. 

First we generate our sample; for a BPR-like model this consists of three lists of user, positive item, negative item triples $(u,i,j)$:
%\todo{ugly text}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
sampleU, sampleI, sampleJ = [], [], []
for _ in range(Nsamples):
    u,i,_ = random.choice(interactions) # positive sample
    j = random.choice(items) # negative sample
    while j in itemsPerUser[u]:
        j = random.choice(items)
    sampleU.append(userIDs[u])
    sampleI.append(itemIDs[i])
    sampleJ.append(itemIDs[j])
\end{lstlisting}
\end{minipage}

Next we must redefine our score function to operate over a sample rather than a single data point. Note that rather than merely iterating over all points, estimates for all samples in our batch are computed using efficient vector operations:

%To do so we first redefine our score (and predict)
%\todo{\texttt}
%functions to operate on a sample

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
    def score(self, sampleU, sampleI):
        u = tf.convert_to_tensor(sampleU, dtype=tf.int32)
        i = tf.convert_to_tensor(sampleI, dtype=tf.int32)
        beta_i = tf.nn.embedding_lookup(self.betaI, i)
        gamma_u = tf.nn.embedding_lookup(self.gammaU, u)
        gamma_i = tf.nn.embedding_lookup(self.gammaI, i)
        x_ui = beta_i + tf.reduce_sum(tf.multiply(gamma_u, gamma_i), 1)
        return x_ui
\end{lstlisting}
\end{minipage}

The `call' function is similarly modified:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
    def call(self, sampleU, sampleI, sampleJ):
        x_ui = self.score(sampleU, sampleI)
        x_uj = self.score(sampleU, sampleJ)
        return -tf.reduce_mean(tf.math.log(tf.math.sigmoid(x_ui - x_uj)))
\end{lstlisting}
\end{minipage}
%\todo{could probably drop this part}
%\todo{Replace with the updated code}

\section{Random-Walk Methods}

Say something about this somewhere?

\section{Beyond a `Black-Box' View of Recommedation}

%\todo{not sure if below argument should go here}
Finally, we should mention that our view of recommendation through the lens of machine learning represents only part of the study of recommender systems. For the most part, we have taken a `black box' view in which we regard the `recommender system' as merely a model that predicts user-item interactions (clicks, purchases, ratings, etc.)~as accurately as possible.

Although high-fidelity prediction is clearly necessary to build a successful recommender, it is only a small part of the picture. For example, we have not considered broader questions of what makes a recommender system `usable' or would ultimately drive user satisfaction or engagement. For example, if a user watches \emph{Harry Potter} should they be recommended its sequel, or another movie from the same genre? The former might maximize some na\"ive metric like click probability, whereas the latter is more likely to generate a suggestion that the user isn't already aware of. But either could be a legitimite goal of building a recommender system: helping a user quickly navigate a user interface by predicting their next interaction is just as important as recommending for novelty or discovery.

Such questions go beyond the black-box supervised learning view of recommendation: they are less questions about how to accurately predict the next action, and more about what we should do with that prediction. At the very least such questions require more nuanced evaluation metrics, if not user studies. While this book largely avoids discussion of user interface design, in \Cref{chap:ethics} we'll revisit the consequences of how recommender systems are applied, and look at strategies to improve personalized recommendation beyond simply optimizing prediction accuracy.


\section{History and Emerging Directions}
%\todo{Probably drop from all sections}

%Early approaches: memory-based to model-based, trend away from rating prediction to implicit feedback models

So far we have attempted to construct something of a narrative behind the development of recommender systems: we started with simple `memory-based' solutions (\Cref{sec:?}), followed by `model-based' approaches such as latent factor models (\Cref{sec:?}); later we argued about the benefits of leveraging implicit feedback (clicks, purchases, etc.)~rather than relying on ratings (\Cref{sec:?}); finally, we began to discuss emerging trends in neural-network based recommendation (\Cref{sec:?}).
%
While this narrative reflects current thinking on the topic, the 
%However the 
actual history of recommender systems is substantially more complicated:
One survey paper
\cite{burke2002hybrid}
%https://link.springer.com/content/pdf/10.1023/A:1021240730564.pdf
points out that even neural-network based recommender systems have been proposed since the early nineties
\cite{jennings1993user}.



To a large extent, research on recommender systems has been driven by the release and adoption of large-scale benchmark datasets. High-profile competitions such as the \emph{Netflix Prize} 
%(including a dataset of 100 million ratings) 
\cite{bennett2007netflix} have driven widespread interest in recommendation problems: the specific nature of the data (purely based on interactions with no side-information); the choice of metrics used (the mean-squared error);
%\todo{style}
and the specific dynamics of the data itself (e.g.~the critical role of temporal dynamics), all show their influence in the models we've explored throughout this chapter. Likewise other datasets and competitions, including industrial datasets (e.g.~\emph{Yelp}, \emph{Criteo}) and (e.g.~\emph{MovieLens} \cite{harper2015movielens})
have inspired models based on alternate settings and evaluation metrics. 

A constant theme in such research is the extent to which new models must be designed to adapt to the specific dynamics of new datasets. 
%which we revisit when exploring models based on temporal dynamics, text, and visual data in later chapters.
%\todo{not great}
As we'll explore in upcoming chapters, recent research on recommender systems has sought to incorporate rich signals in the form of text, temporal and social signals, or images. Such factors serve not only to improve the predictive accuracy of recommendation models, but can also help to make recommendation models more interpretable, and to deal with modalities of data not supported by traditional recommendation approaches.
%Given the variety of textual, temporal, and visual signals, 
%Content rich approaches (extend above para)
We revisit such \emph{content aware} approaches throughout the remainder of this book, as we begin to develop techniques that make use of social (\Cref{chap:contentStructure}), temporal (\Cref{chap:temporalSequential}), textual (\Cref{chap:text}), and visual (\Cref{chap:visual}) signals.

Methodologically, recent research on recommender systems has been dominated by deep learning-based approaches, as we discussed a little in \Cref{sec:?}.
%Development of DL-based approaches
%
Besides models based on multilayer perceptrons, convolutional neural networks, or autoencoders, a major trend has been to incorporate ideas from natural language processing. Roughly speaking, models of natural language are concerned with modeling the semantics of sequences of discrete tokens (i.e.,~words or characters), and thus translate naturally to recommendation problems involving sequences of interaction over a discrete set of items. Recommender systems based on natural language models (e.g.~\emph{Self-Attention},\emph{Transformer},\emph{BERT}, etc.)~arguably represent the current state-of-the-art \cite{kang2018self,sun2019bert4rec}. We'll explore this relationship in \Cref{chap:temporalSequential,chap:text} when developing general-purpose models of sequences and text.
%Development of models from natural language and recent directions

Images, sets, text

A more detailed history of recommender systems is provided in several survey papers
%Summary of existing surveys
\cite{konstan1998recommender}
%https://www.aaai.org/Papers/Workshops/1998/WS-98-08/WS98-08-016.pdf

\cite{burke2002hybrid}


\cite{bobadilla2013recommender}
%http://romisatriawahono.net/lecture/rm/survey/information%20retrieval/Bobadilla%20-%20Recommender%20Systems%20-%202013.pdf


%\todo{finish}

%Development of large benchmark datasets, popular challenges. Movielens, Netflix, Criteo, Yelp, etc., a few more exotic choices

%and have often highli
%including recommender systems that make use of text, social networks, or images.

%Development of content-based approaches, images, text, additional datasets, etc.



\section{Exercises}

\begin{exercises}

\item Modify the Jaccard similarity implementation from \Cref{sec:?} to implement the cosine similarity. Compare the results of the two implementations on the Amazon data.
%\todo{hastily written}

\item The Jaccard similarity-based recommender we implemented in \Cref{sec:?} proved an effective recommender, though our implementation was inefficient. The main source of inefficiency was due to iteration over \emph{all} items. A more efficient implementation might first build smaller a \emph{candidate set} of items, by noting that only those items with at least one user in common with this the query could potentially have non-zero Jaccard similarity. This candidate set can be built by taking all users who have purchased the query $i$, and taking the union over \emph{other} items they have purchased (other than $i$), i.e.,~$\bigcup_{u \in U_i} I_u \setminus \lbrace i \rbrace$. After modifying our implementation from \Cref{sec:jaccard} to use this candidate set, confirm that it produces identical recommendations, and compare its running time to the na\"ive implementation.


\item Implement rating prediction models based on the formulas in \Cref{eq:simPredict,eq:simPredict2,eq:simPredict3}. Compare the three in terms of their Mean Squared Error (using either the entire dataset or a random sample).
%\todo{Which dataset?}

-- Show that the Pearson correlation is related to the $R^2$ coefficient.

-- Some comparison of the various recommender systems error metrics, possibly following the rating prediction experiment above.
%\todo{finish}

\item Before implementing the recommender system described in \Cref{eq:MFOjbective}, it is instructive to implement simpler variants in order to understand the gradient descent procedure. Try implementing a \emph{bias-only} model, i.e.,~one that makes predictions according to $r(u,i) = \alpha + \beta_u + \beta_i$. This can be achieved either by computing derivatives for this simplified model (as we did in \Cref{sec:?}), or more simply by discarding the latent-factor terms from the Tensorflow
%\todo{style}
code from \Cref{sec:?}. Implement this model and compare its performance (in terms of the MSE) to one which always predicts the average rating. When debugging gradient-descent models, it can be instructive to isolate individual terms (i.e.,~updating only a single parameter at a time) to determine that each update results in an improvement of the objective. 

\item In practice the model above 
does little more than measure the overall quality of items (through $\beta_i$) and rating biases of users ($\beta_u$), i.e.,~its predictions are not personalized. Note however that the items with the largest values of $\beta_i$ may not be those items with the highest average ratings.\footnote{Consider why this should be the case. For example, a mediocre item which tends to be rated by `generous' users (high $\beta_u$) could have a high average rating but a low value of $\beta_i$.} Using the model you trained above, find the items with the largest values of $\beta_i$, and compare those to the items with the highest average ratings.



\item Compute the derivatives $\frac{\partial \text{obj}}{\partial \beta_i}$ and $\frac{\partial \text{obj}}{\partial \gamma_{i,k}}$ for the objective from \Cref{eq:MFObjective}.
%\todo{exercise too similar to above}
\item (Hard) In \Cref{sec:latentfactor} we predicted star ratings using a model that optimized the Mean Squared Error. However the ratings we are predicting in many datasets are \emph{integer} valued, e.g.~$r_{u,i} \in \lbrace 1,2,3,4,5 \rbrace$. In light of this, it might be tempting to round the predictions of our model to the nearest integer. In this exercise, we'll show that this type of rounding is not effective, i.e.,~that the rounded predictions will tend to have higher MSEs. For this exercise, suppose that when your model outputs a prediction number $a$, the correct answer is $\lfloor a \rfloor$ with
probability $\lceil a \rceil - a$, and $\lceil a \rceil$ with probability $a - \lfloor a \rfloor$ (e.g.~if it outputs 4.2 then the correct answer is 4 80\% of the time and 5 20\% of the time).
%\todo{Rewrite above, copied from class}
\item aoeu
\end{exercises}

\stepcounter{projectCounter}
\subsection{Project \theprojectCounter: }
\label{proj:2}
%\todo{project title}

In this project we'll build recommender systems to make predictions related to book reviews from \emph{Goodreads}.
Goodreads data. Write introduction. Which dataset, how to split, etc.
%\todo{above}

\begin{enumerate}
    \item First, let's consider using the data to predict ratings. Start by implementing simple neighborhood-based approaches as in \Cref{sec:?}, followed by a model-based approaches as in \Cref{sec:?}. Consider at least the following baselines: (a) the approach of \Cref{eq:?}; (b) the same approach, interchanging users and items; (c) a model-based approach using only an offset $f(u,i) = \alpha$; (d) using only an offset and biases $f(u,i) = \alpha + \beta_u + \beta_i$; (e) using latent factors $f(u,i) = \alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i$.
    %There are several recommendation tasks that you might consider on such a dataset. For example, you could predict what book a user would read next, 
    \item Try to thoroughly tune and regularize the latent factor model you developed above. Some factors you might consider include (a) the number of latent factors $K$; (b) the regularization appoach, for example you might be able to improve performance by using separate regularizers for the bias terms and latent factors, i.e.,~$\lambda_1 \Omega(\beta)$ and $\lambda_2 \Omega(\gamma)$; (c) other factors, such as learning rates, initialization schemes, etc.
    \label{project2part2}
    \item Second, let's consider using the same data for an implicit feedback task, namely to predict whether a user will \emph{read} a book or not. Before solving this using complex approaches specifically designed for implicit feedback, it's worth considering how much progress can be made using simpler approaches based on \emph{classification}. Start by building a training set consisting of all pairs $(u,i)$ of books $i$ that user $u$ has read; next build an (equally-sized) set of \emph{negative} pairs $(u,j)$ of books the user \emph{hasn't} read (e.g.~by sampling randomly). Now, we want to build a feature vector $\phi(u,i)$ that can be used to predict whether a user $u$ has read book $i$ or not. As we discussed in \Cref{sec:?}, to build a useful recommender system, we must include features that describe \emph{interactions} between the user and the book. Examples of features you might use could include:
    \begin{itemize}
        \item The popularity of the book (i.e.,~the number of times item $i$ appears in the training set);
        \item The predicted rating score for the pair $(u,i)$ that you found in Part~\ref{project2part2};
        %\todo{referencing the part is ugly}
        \item The Jaccard similarity (or any other similarity measure) between $i$ and the \emph{most similar} book user $u$ has read (i.e.,~$\max_{j \in I_u} \text{sim}(i,j)$);
        \item Likewise, the similarity between the \emph{user} $u$ and the most similar user who has read $i$;
        \item Any other similarity measures 
    \end{itemize}
\end{enumerate}




\chapter{Content and Structure in Recommender Systems}
\label{chap:contentStructure}
%\todone{Could fold into above chapter, depending on scale. Much too long.}

So far, the systems we've built for personalized recommendation have been based purely on interaction data. We argued in \Cref{sec:?} as to why interactions are often sufficient to capture all of the critical signals that we need. However, our argument that 

\begin{itemize}
    \item Only a limited amount of interaction data may be available. Our argument that interaction data is sufficient to capture subtle preference signals applies only in the limit, i.e.,~when a large number of interactions are available for each user (or item). When few interactions are available (or none, in \emph{cold-start} settings), one must instead rely on user or item \emph{features} to estimate initial preference models.
    \item Beyond improving performance, incorporating features into recommender systems may be desirable for the sake of model interpretability. For example, we may wish to understand how a user will react to a change in price; doing so effectively may require that price features are appropriately `baked in' to the model (we study this specific case in \Cref{sec:?}).
    \item User preferences or item properties may not be \emph{stationary}. Even the simple fact that Christmas movies are unlikely to be watched in July cannot be captured simply by adding more latent dimensions. Although not the topic of this chapter, we'll revisit models of such temporal and sequential dynamics in \Cref{chap:temporalSequential}.
    \item Many settings simply do not follow the setup we developed in \Cref{chap:recsys}. For example, many recommendation scenarios have a social component (dating, bartering, etc.)~that must be accounted for in addition to user-to-item compatibility.
\end{itemize}
%\todo{not the best examples above}

In this chapter we'll develop models that help us to adapt to the situations above. First, we'll explore general-purpose strategies to incorporate \emph{features} into recommender systems, starting with Factorization Machines in \Cref{sec:factorization}. For the most part, our goal is to study strategies to incorporate simple numerical and categorical features; we develop strategies for the specific cases of temporal, textual, and visual features in \Cref{?,?,?}. 
We are especially interested in how features can be incorporated for the sake of solving so-called \emph{cold-start} problems (\Cref{sec:?}), whereby we have little (or no)
%\todo{check if I've already said the above recently}
data associated with new users or items, as must infer an initial model of their preferences.

In this chapter we'll also explore various modaliteis of recommendation that don't exactly follow the basic setup from \Cref{chap:recsys}. We'll explore examples including online dating (\Cref{sec:?}), bartering (\Cref{?}), social and group recommendation (\Cref{?}), among others. 
%\todo{update list}
In exploring such settings, our goal is not only to explore a few specific applications of interest, but more importantly to understand the overall process of designing and adapting personalized machine learning techniques for situations that don't perfectly align with traditional recommendation settings.


\section{The Factorization Machine}
\label{sec:factorization}

Factorization Machines \cite{rendle2010factorization} are a general-purpose approach that seeks to incorporate general features into models that capture pairwise interactions.

In essence, the factorization machine extend the approach behind the latent factor model (\Cref{sec:latentfactor}). The latent factor model embeds users and items into low dimensional space via $\gamma_u$ and $\gamma_i$, and then models the interaction between them via an inner product; the factorization machine extends this approach to incorporate arbitrary pairwise interactions between users, items, and other features.

The input to the model is a feature matrix $X$ and a target $y$. In the simplest case, $X$ might simply encode the identity of the user and item via a one-hot encoding, though can be extended to incorporate any additional properties associated with the interaction:
\begin{equation}
{
\left[
    \begin{tabular}{c}
    1000000 $\ldots$ 000100000 $\ldots$ 0001000 $\ldots$ 15.95\\
    0001000 $\ldots$ 000000010 $\ldots$ 0001000 $\ldots$ 12.25\\
    0100000 $\ldots$ 000100000 $\ldots$ 0000010 $\ldots$ 15.00\\
    0000100 $\ldots$ 010000000 $\ldots$ 0010000 $\ldots$ 17.50\\
    1000000 $\ldots$ 000000010 $\ldots$ 1000000 $\ldots$ 19.95\\
    $\smash{\underbrace{0000100}_{\text{user}}}\vphantom{0}$ $\ldots$ 
    $\smash{\underbrace{000010000}_{\text{item}}}\vphantom{0}$ $\ldots$
    $\smash{\underbrace{0000010}_{\text{weekday}}}\vphantom{0}$ $\ldots$
    $\mathclap{\smash{\underbrace{\hphantom{0}}_{\mathrlap{\text{price}}}}}1$
    \end{tabular}
\right]
}
%\todo{Make above not ugly}
\label{eq:FMinteractionMatrix}
\end{equation}

The basic idea of the factorization machine is then to model \emph{arbitrary} interactions between features. Each feature dimension is associated latent representation $\gamma_i$; the model equation is then defined in terms of all \emph{pairs} of (non-zero) features:
\begin{equation}
    f(\mathbf{x}) = \underbrace{w_0 + \sum_{i=1}^F w_i x_i}_{\text{offset and bias terms}} + \underbrace{\sum_{i = 1}^n \sum_{j = i+1}^n \langle \gamma_i, \gamma_j \rangle x_i x_j}_{\text{feature interactions}}.
    %\todo{Summation goes up to n (here and below), but I don't think I ever defined n?}
    \label{eq:FMequation}
\end{equation}

It is instructive to consider the case where the interaction matrix in \Cref{eq:FMinteractionMatrix} includes \emph{only} a user and item encoding. In such a case, \Cref{eq:FMequation} expands to be identical to the latent factor model from \Cref{eq:latentFactor?}, i.e.,~the only interaction term is $\gamma_u \cdot \gamma_i$ for a user $u$ and item $i$. 

As such, the Factorization Machine can be viewed as a generalization of a latent factor model, that allows for additional types of interactions to be considered. For example, if we include an additional one-hot feature in \Cref{eq:FMinteractionMatrix} that encodes the \emph{previous} item the user consumed, then the factorization machine
%\todo{capitalization?}
will include an expression encoding the compatibility of the next item with the previous one, i.e.,~the model can learn how \emph{contextually relevant} the previous item is compared to the next one. It is useful to compare this approach to the models we design specifically to handle sequential inputs in \Cref{sec:sequential}. \cite{rendle2010factorization} discusses such topics, describing the extent to which the general-purpose factorization machine formulation subsumes various approaches designed to handle specific types of features.

\cite{rendle2010factorization} describes how the model equation of \Cref{eq:FMequation} can be computed efficiently (and how parameter learning can be done efficiently), by showing that the interaction term can be rewritten as
\begin{equation}
    \sum_{i = 1}^n \sum_{j = i + 1}^n \langle \gamma_i, \gamma_j \rangle x_i x_j = \frac{1}{2} \sum_{f = 1}^k \left( \left( \sum_{i = 1}^n \gamma_{i,f} x_i \right)^2 - \sum_{i = 1}^n \gamma_{i,f}^2 x_i^2 \right),
\end{equation}
which allows for computation that is $O(kn)$ (i.e.,~the dimension of the latent factors multiplied by the feature dimensionality).


\subsection{Factorization Machines in Python with \emph{FastFM}}

%As we shall see later, 
As we saw above, the Factorization Machine is a highly flexible, general-purpose technique to incorporate numerical or categorical 
%As we shall see later, the factorization machine is a highly flexible approach that can 
features into recommender systems. Later in this chapter, and in \Cref{chap:temporalSequential} we'll explore specific types of dynamics
%\todo{make sure above is true}
that can be captured via Factorization Machines, but for the moment we'll explore an implementation of a `vanilla' Factorization Machine via the \emph{FastFM} library \cite{bayer2016fastfm}.
%\todo{Say what dataset it is?}

First, we read our dataset and construct a mapping from each user to a specific index (from $0$ to $|U| - 1$);
%\todo{notation}
this index will be used to associate each user and item with a feature dimension in our one-hot encoding (as in \Cref{eq:FMexamp}):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
userIDs,itemIDs = {},{}
for d in data:
    u,i = d['user_id'],d['book_id']
    if not u in userIDs: userIDs[u] = len(userIDs)
    if not i in itemIDs: itemIDs[i] = len(itemIDs)
    
nUsers,nItems = len(userIDs),len(itemIDs)
\end{lstlisting}
\end{minipage}
%\todo{No import statement? What do I normally do?}

Next, we build our matrix of features associated with each interactions. Each feature is simply the concatenation of a (one-hot encoding of) a user ID and an item ID. Note that we use a sparse data structure (\texttt{lil\_matrix})
%\todo{Do I use texttt elsewhere?}
to represent the feature matrix. Although we only use user and item IDs here, these feature vectors could straightforwardly be extended to include other features, such as those we explore later in the chapter:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = scipy.sparse.lil_matrix((len(data), nUsers + nItems))
for i in range(len(data)):
    user = userIDs[data[i]['user_id']]
    item = itemIDs[data[i]['book_id']]
    X[i,user] = 1
    X[i,nUsers + item] = 1

y = scipy.array([d['rating'] for d in data])
\end{lstlisting}
\end{minipage}

Finally, we split the data into training and test fractions, fit the model, and compute its predictions on the test set:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X_train,y_train = X[:2000000],y[:2000000]
X_test,y_test = X[2000000:],y[2000000:]

fm = fastFM.als.FMRegression(n_iter=1000, init_stdev=0.1, rank=2, l2_reg_w=0.1, l2_reg_V=0.5)

fm.fit(X_train, y_train)
y_pred = fm.predict(X_test)
\end{lstlisting}
\end{minipage}
%\todo_{Explain above parameters}
%https://ibayer.github.io/fastFM/tutorial.html

The model has several tunable parameters, including

\section{Cold-Start Recommendation}

So far, all of the recommendation approaches we have developed have depended on having detailed \emph{interaction histories} associated with users and items. Naturally, we cannot find similar users (as in \Cref{sec:neighborhood}) to a user who has no purchase history; likewise, we cannot estimate latent parameters $\gamma_u$ (or even a bias $\beta_u$) for a user who has never rated or purchased any items (as in \Cref{sec:latentFactor}).

As such, we need to develop recommendation approaches that can be useful in so-called \emph{cold-start} scenarios. Depending on the setting, either users or items may be `cold' (i.e.,~have zero associated interactions). 

We'll investigate two categories of approach to deal with cold-start problems. First, one may attempt to deal with cold-start situations via the use of \emph{side-information} about users or items. Side information could range from product images, to text, or social interactions. In each case,
%(including more complex approaches we develop later in the book), 
side information gives clues as to the properties of an item, whether by learning user preferences toward observed item features (\Cref{sec:coldStartSide}),
harvesting weaker signals such as the preferences of a user's friends (\Cref{sec:social?}), 
or by using item features to estimate item latent factors (\Cref{sec:vbpr}).
%\todone{not great}
We explore some of the simpler methods below, but revisit the use of side-information throughout the book.
Second, we'll explore methods that directly seek to 
elicit 
%\todo{elicit or solicit?}
preferences from new users through surveys (\Cref{sec:?}).

\subsection{Addressing Cold-Start Problems with Side Information}
\label{sec:coldStartSide}
%Pairwise Preference Regression for Cold-start
%Recommendation (feature based)

In the absence of historical interaction data associated with users or items, one option is to resort to secondary signals. 
\cite{park2009pairwise} considers cold-start settings in the context of movie recommendation. For movies, associated features are available such as the release year, genre, (etc.),~which can be encoded (for example) as a one-hot vector. For users, demographic features are used such as a user's age, gender, occupation, or location. These features are captured via user and item feature vectors $x_u$ and $z_i$ for each user $u$ and item $i$.

Recall that at the beginning of \Cref{chap:recsys}, we discussed the differences between recommendation and regression and argued that recommendation was fundamentally different from simple linear regression on user and item features. Critically, we argued that recommender systems must model the \emph{interaction} between users and items, in order to be able to meaningfully personalize predictions for each user.

In order to capture interactions, \cite{park2009pairwise} use what is termed a \emph{bilinear} model, to estimate the compatibility between user and item features. The model parameters can then be described via a matrix $W$, and user-item compatibility can be written as
\begin{equation}
    s_{u,i} = x_u W z_i^T = \sum_{a = 1}^{|x_u|} \sum_{b = 1}^{|z_i|} x_{u,a} z_{i,b} W_{a,b},
\end{equation}
Here, unlike the linear regression model from (e.g.)~\Cref{eq:whyRecDifferent}, $W$ now encodes how user features should \emph{interact} with item features. That is, a parameter $W_{a,b}$ encodes the extent to which the $a^{\text{th}}$
%\todo{consistent way of writing this}
user feature is compatible with the $b^{\text{th}}$ item feature. So, the model can learn (for example) the extent to which users in the 35-50 demographic will respond positively to the teen romance genre.

Both $x_u$ and $z_i$ include a constant feature. These features (or rather the corresponding entries in $W$) roughly fill the role of bias terms (i.e.,~$\alpha$, $\beta_u$ and $\beta_i$ in \Cref{eq:latentSomething?}), that is, they allow the model to learn the extent to which users in a certain demographic, or movies of a certain genre, tend to yield higher or lower ratings than others. 

The model is trained so that the compatibility $s_{u,i}$ should align with observed interactions (e.g.~ratings). \cite{park2009pairwise} achieves this using a specific type of pairwise loss (i.e.,~a loss that considers two items at a time, similar to the BPR loss of \Cref{eq:bpr??}), though this is an implementation detail that is not critical to the main idea of the method.

Ultimately, the method is evaluated on two movie datasets (\emph{MovieLens} and \emph{EachMovie}). Cold users and items in these datasets are simulated, simply by withholding interactions from a subset of users and items at training time, with interactions from those users being used to evaluate the system at test time. Experiments show that when considering cold users and/or items, the method outperforms alternatives that don't make use of features.

\paragraph{Other Types of Cold-Start Recommendation}
%\todo{Capitalize paragraph-level headings?}

Obviously there is significant room to improve upon the above system, for example we might desire a model that simultaneously leverages features for cold users, and interactions for non-cold users, and learns to smoothly transition between the two groups.

We'll revisit the topic of cold-start recommendation regularly in later chapters, as well as `cool-start' settings (where we have only a few interactions per user or item), as we develop systems that operate on features from sequences, text, and images. Whether explicitly designed for cold-start or not, such methods often seek to use side information to circumvent the paucity of available interaction data.

\subsection{Addressing Cold-Start Problems with Surveys}

An alternative to relying on side information in user cold-start settings is simply to directly solicit the preferences of new users once they first interact with the recommender system.

\cite{rashid2002getting} investigated strategies for generating initial user `surveys,' in order to most quickly learn the preference dynamics of new users


Functional Matrix Factorizations for Cold-Start
Recommendation (interview based) \cite{zhou2011functional}

\section{Multisided Recommendation}

So far, our view of recommendation and personalization has consisted of maximizing some predicted utility for each user, e.g.~estimate their ratings or which items they'll interact with. Furthermore, every user has predictions made independently of each other.

Such a setting seems natural when considering contexts such as movie recommendation, but there are a variety of cases where such models are inappropriate. For example, recommendation on an online dating platform would require quite different assumptions. For instance, the problem has symmetries in the sense that the users being recommended are also receiving recommendations---as such, users must be interested in their matches, but the matches must have a reasonable chance of reciprocating. Likewise we must ensure not only that everyone receives recommendations, but that everyone \emph{is recommended to somebody}.

These types of problems are referred to as \emph{multisided} or \emph{multistakeholder} recommendations~\cite{abdollahpouri2017recommender}. Such constraints appear in many settings, many of which we will visit throughout the book. In \Cref{sec:groupRec} we will look at \emph{group} recommendation, where recommendations must simultaneously satisfy the interests of multiple users in a group. And in \Cref{chap:ads}, we will consider advertising settings, where we must consider not only user preferences but also the budgets of individual advertisers (which prevent us from recommending the most compatible ads to everyone). Finally we'll revisit the topic in depth in \Cref{sec:improving}, where we consider issues of fairness, calibration, balance, etc. For example, when recommending movies we might want reasonable coverage of different genres \cite{steck2018calibrated}, or when recommending authors, we might want our recommendations not to be too narrow in terms of gender or nationality \cite{ekstrand2018exploring}.

For now, we'll consider two specific examples of multistakeholder recommendation, namely online dating and bartering (i.e.,~recommending trading partners).

%Multistakeholder Recommendation
%\cite{abdollahpouri2017recommender}

\subsection{Online Dating}

\cite{pizzato2010recon} studied recommendation in the context of online dating. Online dating has several constraints not present in the types of recommendation problems we've seen so far, in particular due to the fact that the users being recommended are the same ones receiving recommendations. 

\cite{pizzato2010recon} considers the specific objective of \emph{reciprocal communication}, which is partly motivated by a specific mechanism in the data they study. That is, a recommendation of a user $v$ to a user $u$ should be considered successful only if $u$ messages $v$ \emph{and} $v$ responds to their message.

Actual compatibility scores $f(u,v)$ in \cite{pizzato2010recon} are estimated using a fairly simple feature-based strategy that looks for a match between $u$'s preferences and $v$'s attributes (some of which may be matched strictly, e.g.~if a user has historically expressed interest in only a certain gender).  Following this the \emph{reciprocal compatibility} is simply the harmonic mean of the two compatibility scores:
\begin{equation}
    \text{reciprocal compatbility}(u,v) = \frac{2}{f(u,v)^{-1} + f(v,u)^{-1}}.
\end{equation}
The harmonic mean here is preferable to (e.g.)~the arithmetic mean as it does not allow either user's preference to `dominate' the compatibility estimate, i.e.,~two users are only compatible if \emph{both} have high compatibility scores for each other.


Beyond the notion of reciprocity considered in \cite{pizzato2010recon}, online dating has certain `balance' or `diversity' constraints not yet seen in other problems, for example we cannot identify a user with `ideal' characteristics and recommend them to everyone (which might be perfectly reasonable for, say, movies); instead, the system only has utility if users both receive, and appear in, recommendations.
%\todo{Moved this paragraph, doesn't really fit}
%\todo{Mention we'll see these types of constraints later, e.g. in diversity and advertising}

\subsection{Bartering Platforms}

\cite{rappaz2017bartering} considered the problem of generating recommendation systems for \emph{bartering} platforms, i.e.,~settings in which users exchange goods.

They study several settings in which products are exchanged, including CDs and DVDs, though most of their analysis centers around three datasets, of books (from \emph{bookmooch.com}), beers (from \emph{ratebeer.com}), and video games (from \emph{reddit.com/r/gameswap}).

On each of these websites, users have both a `wishlist' $W_u$ and a `give-away' list $G_u$, i.e.,~sets of items they wish to give or receive. Given this constraint on the problem, one might think that recommending compatible trades is as simple as identifying compatible pairs. However surprisingly the data reveals that `eligible' swapping partners are incredibly rare, and the vast majority of logged trades occur between items that were not expressly included in a user's wishlist; thus there is a need to build a system that can model likely trading partners via latent preferences. They also note that users repeatedly trade with the same partners, indicating that there is a social component to trading.

Given the above two factors, the basic model combines a standard latent factor representation with a social term. Given a user $u$, an item $i$, and a potential trading partner $v$, their compatibility is modeled as
\begin{equation}
    f(u,v,i) = \gamma_u \cdot \gamma_i + S_{u,v}.
\end{equation}
Here $\gamma_u$ and $\gamma_i$ are low-rank factors as in \Cref{eq:latentFactor?}, whereas $S_{u,v}$ is a (potentially full-rank) matrix $S \in \mathbb R^{|U|\times |U|}$;
%\todo_{is the number of users |U|?}
although $S$ potentially encodes a large number of parameters, in practice it is very sparse as the number of observed trading partners is limited.

Note that the above model captures only the interest of one user toward another's item; to model reciprocal interest, \cite{rappaz2017bartering} simply captures the average of interest in both directions (\Cref{fig:bartering}):
\begin{equation}
    f(u,i,v,j) = \frac{1}{2}(f(u,v,i) + f(v,u,j)).
\end{equation}
Other aggregation functions besides the arithmetic mean (such as the harmonic mean) can be used, though the arithmetic mean proved the most effective, indicating that a weak preference from one user can be made up for by a strong preference from another. The model also includes a temporal term encoding timepoints when certain users are particularly active and certain items are particularly popular, though we leave discussion of temporal models to \Cref{chap:temporal}.

Ultimately, the method is evaluated in terms of its ability to assign higher scores to observed interactions compared to non-observed ones (i.e.,~using a BPR-like training and evaluation scheme, as in \Cref{sec:BPR?}). The main conclusion of the experiments is that several components are important in bartering settings: reciprocal interest, a social history of trades, as well as temporal `trends' across users and items.

\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
%  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=8mm]
%\node[fsnode,on chain,draw,circle] (f1) [label=left: 1, label={[label distance=0.7mm]90:women:}] {};
\node[fsnode,on chain,draw,circle] (f1) [label=left: $u_1$] {};
\node[fsnode,on chain,draw,circle] (f2) [label=left: $u_2$] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=8mm]
%\node[ssnode,on chain,draw,circle] (s1) [label=right: 1, label={[label distance=2mm]90:men:}] {};
\node[ssnode,on chain,draw,circle] (s1) [label=right: $i_1$] {};
\node[ssnode,on chain,draw,circle] (s2) [label=right: $i_2$] {};
\end{scope}
%\todo{Hardcoded vertical spacing of men/women NOT CORRECT: label}

% \node[above=of f1] (w1) {women};
% \node[above=of s1] (m1) {men};

\draw[thick] (s1) edge[-] (f1);
% \node [pos=0.66,above=s1] {Transpose};
\draw[dotted,thick] (s1) edge[-] (f2);
\draw[dotted,thick] (s2) edge[-] (f1);
\draw[thick] (s2) edge[-] (f2);




% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}

    \caption[Basic idea behind reciprocal interest; in bartering settings a strong preference from one user compensates for a potentially weaker one from the other.
    %\todo{figure}
    ]{Basic idea behind reciprocal interest, from \cite{rappaz2017bartering}; in bartering settings a strong preference from one user compensates for a potentially weaker one from the other.
    %\todo{figure}
    \label{fig:reciprocalInterest}
    }

\end{figure}
%\todo{Make this figure}

See other examples from multisided fairness paper



\section{Music}

Thesis: Music recommendation and discovery in the long tail
%http://www.mtg.upf.edu/static/media/PhD_ocelma.pdf

%\todo{Does this belong here, or should put it in sequential recommender systems section? Or incorporate audio features into the visual recommendation chapter?}

Features derived from music

Interaction patterns such as skips and completed listens

\section{Group- and Socially-Aware Recommendation}

%\todo{Could also add bundling here, if it's cast more as set-based recommendation}

\subsection{Socially-Aware Recommendation}

Several approaches have sought to incorporate signals from social networks to improve recommendation. The basic idea behind doing so is that social connections will help us to circumvent \emph{sparsity} issues in interaction data. That is, even if a user has only a small number of observed interactions, we can (to some degree) leverage the interactions of their friends, whose opinions they are likely to trust.

Conceptually, the typical approach behind socially-aware recommendation is to use social connections as a form of \emph{regularizer}, which states that a user's preferences should be \emph{similar to} those of their connections in a social network. For example, given a user with few interactions, we might assume that their preferences align with the (average of) their friends; this is a possibly better assumption than the regularizer of \Cref{eq:latentreg3?}, which in practice will essentially discard user latent factors ($\gamma_u$) for users with few interactions.


%SoRec: Social Recommendation Using
%Probabilistic Matrix Factorization
An early attempt to incorporate social networks into recommender systems extended the basic framework of a latent factor model \cite{ma2008sorec}. They looked at data from \emph{Epinions}, which in addition to interaction data in the form of ratings (much like \Cref{eq:interactionMatrix?}), includes a network of `trust' and `distrust' relationships. Unlike a typical social network, these are \emph{signed} relationships, whereby a user explicitly indicates that they `trust' (1) or distrust (-1) another user. That is, in addition to our interaction matrix we have a (directed) adjacency matrix:
\begin{equation}
    A = \underbrace{\left[ \begin{tabular}{ccccc}
    1 & $\cdot$ & $\cdot$ & -1 & 1\\
    $\cdot$ & 1 & -1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 1 & 1 & -1 & $\cdot$\\
    1 & $\cdot$ & 1 & $\cdot$ & 1\\
    -1 & -1 & $\cdot$ & 1 & 1\\
    \end{tabular} \right]}_{\text{users}} \left. \clap{\begin{tabular}{c} \\ \\ \\ \\ \\ \end{tabular}} \right\rbrace \text{\normalsize users}.
    %\todo{In other equations like this, make sure the underbrace text is the right size}
    \label{eq:trustMatrix}
\end{equation}
Ultimately though, the distrust relationships are not used in \cite{ma2008sorec}, as it is argued that the semantics of `distrust' are somewhat more complex than (e.g.)~users having different preference dimensions.

Thus, give a rating matrix $R$, and an adjacency matrix $A$, we want to predict ratings in $R$ in such a way that $A$ informs us about each user's likely latent preferences. The basic idea is to make use of a \emph{shared parameter} $\gamma_u$ for each user. For rating data, $\gamma_u$ is no different from a user latent factor in a matrix factorization model, i.e.,~it is combined with an item latent factor and used to predict the rating, in this case via a sigmoid function:\footnote{The specific choice of the sigmoid function is an implementation detail, and ratings are scaled to be in the range $[0,1]$ to accommodate this choice.}
\begin{equation}
    r_{u,i} = \sigma(\gamma_u \cdot \gamma_i).
\end{equation}
Next, the parameter $\gamma_u$ is \emph{re-used} to predict trust relationships in $A$:
\begin{equation}
    a_{u,v} = \sigma(\gamma_u \cdot \gamma'_v).
\end{equation}
The original paper allows $A$ to be a weighted matrix, indicating varying degrees of social trust, but for simplicity we assume here that $A$ contains only trust (1) and not-trust (0) values.

Note that while $\gamma_u$ is a shared parameter, $\gamma'v$ is not; since the matrix $A$ is directed, $\gamma_u$ can be thought of as explaining why $u$ trusts others, whereas $\gamma'_v$ explains why $v$ \emph{is trusted by} others.
%\todo{confusing, does the above help?}

In practice we are usually not interested in predicting entries $a_{u,v}$; rather, the trust relationships are additional data that should help us to calibrate $\gamma_u$ more efficiently. For example, if a user has few observed ratings, $\gamma_u$ can still be estimated via their trust relationships, giving us an initial estimate of their rating behavior in cold- (or cool-)start settings.
%\todo{Did I already make this argument above?}

The overall objective then takes the form
\begin{equation}
    \underbrace{\sum_{(u,i) \in R} (r_{u,i} - \sigma(\gamma_u \cdot \gamma_i)^2}_{\text{rating prediction error}} + \lambda^{(\text{trust})}\underbrace{\sum_{(u,v) \in A} (a_{u,v} - \sigma(\gamma_u \cdot \gamma'_v)^2}_{\text{trust prediction error}} + \lambda\| \gamma \|_2^2,
\end{equation}
%\todo{Parentheses don't add up in above equation}
where $\lambda^{(\text{trust})}$ trades-off the importance of predicting the trust network. Ultimately, experiments in \cite{ma2008sorec} show that the trust network helps to predict ratings more accurately than matrix factorization alone. Of course, it should be noted that trust relationships on \emph{epinions} 
%\todo{capitalization, italicization?}
are very closely tied to opinion dimensions, presumably moreso than in other social networks.

Note that the above is essentially a more complex form of cold-start (or `cool-start')
%\todo{make sure not to say the above too many times}
recommendation (as we saw in \Cref{sec:coldstartRec}), in the sense that we are leveraging a form of side-information (social connections) to make up for a paucity of interaction data. In the case of a user who has never rated an item (but has social connections), the system can still reasonably estimate $\gamma_u$ from the preference dimensions of $u$'s friends.


\subsection{Social Bayesian Personalized Ranking}

%Leveraging Social Connections to Improve Personalized
%Ranking for Collaborative Filtering

Above we showed how matrix factorization frameworks can be extended to incorporate signals from social networks. The intuition behind the idea simply stated that users are likely to have preference dimensions aligned with other users who they `trust.'

Next, we'll see how this idea can be adapted to predict interactions (rather than ratings), by incorporating social connections into the Bayesian Personalized Ranking framework from \Cref{sec:BPR}.

Conceptually, using social links to predict
%Personalized ranking 
interactions relies on
a possibly different assumption than we made above. Whereas our previous intuition above was based on some notion of trust, here we are simply assuming that a user is more likely to interact with items (e.g.~to watch movies or read books) if their friends have previously interacted with them.

\cite{zhao2014leveraging} attempted to adapt the assumptions made by Bayesian Personalized Ranking (BPR) to datasets involving social connections. Recall that BPR makes the assumption that a user's compatibility with items they've interacted with ($x_{u,i}$) should be higher than their compatibility with items they haven't interacted with ($x_{u,j}$), which is captured via a sigmoid function:
\begin{equation}
    x_{u,i} \geq x_{u,j} \quad \rightarrow \quad \sigma(x_{u,i} - x_{u,j}) \text{\ should be maximized}
\end{equation}
(see \Cref{eq:BRPOpt1?}). To adapt this to settings involving a social network, \cite{zhao2014leveraging} assumes a third type of feedback: for a user $u$, in adition to \emph{positive} feedback $i$, and \emph{negative} feedback $j$ (as with BPR), we also have \emph{social} feedback $k$, which consists of items consumed by $u$'s connections in a social network.

\cite{zhao2014leveraging} tests two assumptions about how this social feedback should be incorporated. The first states that social interactions are weaker than positive interactions, but still stronger than negative interactions, essentially stating that users are somewhat more likely to interact with items their friends have interacted with:
\begin{equation}
    \underbrace{x_{u,i}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,k}}_{\mathclap{\text{social}}}; \quad \underbrace{x_{u,k}}_{\mathclap{\text{social}}} \geq \underbrace{x_{u,j}}_{\mathclap{\text{negative}}}.
    \label{eq:sbpr1}
\end{equation}
An alternate hypothesis states the opposite: if our friends have interacted with an item but we haven't, this might instead be a signal that we know about the item, but deliberately chose not to interact wit it; in this instance we drop the second assumption from \Cref{eq:sbpr1} and replace it with a weaker assumption:
\begin{equation}
        \underbrace{x_{u,i}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,k}}_{\mathclap{\text{social}}}; \quad \underbrace{x_{u,i}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,j}}_{\mathclap{\text{negative}}}.
    \label{eq:sbpr2}
\end{equation}
Note that neither of these assumptions is `better' than the other; rather they are simply hypotheses that must be tested by determining which best fits real datasets.

To train the model, a BPR-like objective (\Cref{eq:bprOpt1?}) is used, but which now involves two terms. E.g.~using the assumption from \Cref{eq:sbpr1}:
\begin{equation}
    \sum_{(u,i,k) \in \mathcal T} \ln \sigma(x_{u,i} - x_{u,k}) + \sum_{(u,k,j) \in \mathcal T} \ln \sigma(x_{u,k} - x_{u,j}) + \| \gamma \|_2^2,
\end{equation}
where $\mathcal T$ is a training set consisting of positive, negative, and social feedback ($i$, $j$, and $k$) for each user $u$.

Ultimately, \cite{zhao2014leveraging} shows that both models outperform alternatives that don't leverage social connections. Several datasets besides \emph{epinions}
%\todo{italicize, capitalize?}
are used, including data from \emph{Ciao}, \emph{Delicious}, and \emph{LibraryThing} (product reviews, social bookmarks, and books).
Overall, the assumption from \Cref{eq:sbpr2} slightly outperforms that of \Cref{eq:sbpr1} on all datasets.

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Comparison of socially-aware recommendation techniques.}
\label{tab:socialRec}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{lXX}
% \hrule\\
\textbf{Reference} & \textbf{Dataset} & \textbf{Description}\\ 
\cite{ma2008sorec} & Social trust data from \emph{epinions} & Trust links help to regularize $\gamma_u$, which must simultaneously explain rating and trust factors.\\
\cite{zhao2014leveraging} & \emph{epinions}, \emph{Ciao}, \emph{Delicious}, \emph{LibraryThing} &
Friends' interactions act as additional \emph{implicit signals} for recommendation.\\
% \hrule
\cite{o2001polylens} & \emph{MovieLens} & Studies the desirable characteristics of good interfaces for group recoommendation.\\ 
\cite{amer2009group} & \emph{MovieLens} & Designs measures of \emph{group consensus} and proposes recommendation algorithms to maximize them.\\
\cite{pan2013gbpr} & \emph{MovieLens}, \emph{Netflix} & Treats group preferences as weak signals to design pairwise sampling strategies for BPR.
\end{tabularx}
\end{minipage}
\end{table}
%\todo{table formatting}

\subsection{Group-Aware Recommendation}

Somewhat related to the topic of social recommendation is the idea of \emph{group} recommendation, where recommendations should be made collectively to a group of users, rather than to an individual. However, unlike social recommendation, where the goal is normally to use social signals to improve the accuracy of recommendations each individual receives, the goal of group-aware recommendation is generally to produce recommendations that mutually satisfy a group of individuals.

% PolyLens: A Recommender System for
% Groups of Users (basically just has an interface for group recommendations)

Early work on this topic includes \emph{PolyLens}
%\todo{italics}
\cite{o2001polylens}, though this work was mostly concerned with designing user interfaces for the purpose of group recommendation, rather than using group data to improve recommendation performance. Although focused on interface-building, the work does show the utility of group-based recommenders, which could help users to find items that are mutually compatible with the interests of all group members.
%\todo{is that what it actually says?}

Later, \cite{amer2009group} attempted to formalize the notion of group recommendation, by defining useful objectives that define how compatible a set of items is with a group. Given a pre-defined compatibility function $f(u,i)$ (e.g.~the output of a latent factor model), a simple attempt to define group compatibilty between a group of users $\mathcal G$ and an item $i$ might consist of computing 
\begin{equation}
    \text{Average compatibilty of $i$:} \quad \mathit{rel}(\mathcal G, i) = \frac{1}{|\mathcal G|} \sum_{u \in \mathcal G} f(u,i).
    \label{eq:groupRel1}
\end{equation}
Alternately, we could define the compatibility as that of the \emph{least} compatible user in the group, known as \emph{least misery}:
\begin{equation}
\text{Least misery:} \quad \mathit{rel}(\mathcal G, i) = \min_{u \in \mathcal G} f(u,i).
    %2) Least-Misery: rel(G, i) = Min(relevance(u, i))
    \label{eq:groupRel2}
\end{equation}
The latter is preferable in settings where users have constraints, e.g.~to avoid recommending a steakhouse to a group of users, some of whom are vegetarian \cite{amer2009group}.

\cite{amer2009group} argues that in addition to maximizing relevance (or minimizing misery), it is also important that the group has some degree of \emph{consensus} about the quality of an item. That is, users in a group should not drastically disagree about $f(u,i)$, separately from their actual scores. Two disagreement functions proposed are average pairwise disagreement:
\begin{equation}
    \mathit{dis}(\mathcal G, i) = \frac{2}{|\mathcal G|(|\mathcal G| - 1)} \sum_{(u,v) \in \mathcal G, u \neq v} (| f(u,i) - f(v,i) |),
    \label{eq:groupDis1}
\end{equation}
and disagreement variance:
\begin{equation}
    \mathit{dis}(\mathcal G, i) = \frac{1}{|\mathcal G|} \sum_{u \in \mathcal G} \Bigl(f(u,i) - \underbrace{\frac{1}{|G|} \sum_{v \in \mathcal G} f(v,i)}_{\mathclap{\text{average compatibility in $\mathcal{G}$}}}\Bigr)^2.
    \label{eq:groupDis2}
\end{equation}

Finally, \cite{amer2009group} argues that a group \emph{consensus function} should be a combination of both of these factors, i.e.,~a relevance function (\Cref{eq:groupRel1} or \Cref{eq:groupRel2}):
%\todo{maybe use lighter-weight equation references within parentheses?}
\begin{equation}
    \mathcal{F}(\mathcal G, i) = w_1 \times \mathit{rel}(\mathcal G, i) + w_2 \times (1 - \mathit{dis}(\mathcal G, i)),
\end{equation}
where $w_1$ and $w_2$ trade-off the relative importance of the two terms. \emph{Group recommendation} then consists of finding suitable items (e.g.~movies that no user in the group has seen) that maximize a (tuned) group consensus function.

Beyond defining these notions of group consensus, \cite{amer2009group} show how to efficiently select items that maximize the above criteria (note the large number of comparisons involved when performing optimization na\"ively).
%
They also demonstrate experimentally (via a \emph{Mechanical Turk}-based user study of movie recommendations) that both \emph{relevance} and \emph{disagreement} are simultaneously important to achieve satisfaction within a group.


\subsection{Group Bayesian Personalized Ranking}
%\todo{could be paragraph, not much to say here}

Much like \cite{zhao2014leveraging} incorporated social links into Bayesian Personalized Ranking by treating friends' interactions as additional implicit signals, 
`Group BPR' \citep{pan2013gbpr}
seeks to treat group preferences as a form of implicit signal that can be used within a BPR framework.
%  GBPR: Group preference based bayesian
% personalized ranking for one-class collaborative filtering \cite{pan2013gbpr}


While BPR assumes that a user $u$'s compatibility with an observed interaction $i$ is greater than their compatibility with an unobserved interaction $j$ (i.e.,~$x_{u,i} > x_{u,j}$, as in \Cref{eq:bpr???}),
%\todo{compare notation to BPR section}
\cite{pan2013gbpr} assumes that \emph{group} preference acts as a similar form of implicit feedback.
As with Social BPR (\Cref{sec:?}), the goal Group BPR is essentially to leverage weak signals from related users as a way of harvesting implicit pairwise preference feedback.

%\todo{Make sure various acronyms are defined, probably put them in notation page}

Specifically, if a group of users $\mathcal G$ has interacted with some item $i$, their \emph{mutual preference} toward the item, defined as
\begin{equation}
    x_{\mathcal G, i} = \frac{1}{|\mathcal G|} \sum_{g \in \mathcal G} x_{g,i}
\end{equation}
is assumed to be greater than a user $u$'s preference toward an unseen item $j$ (i.e.,~$x_{u,j}$). This notion of group preference is combined with the pairwise preference model from \Cref{eq:bpr???}, resulting in a preference model of the form:
\begin{equation}
    \rho x_{\mathcal G, i} + (1 - \rho) x_{u,i} >  x_{u,j},
    \label{eq:gbprSample}
\end{equation}
where $\rho$ is a hyperparameter controlling the relative importance of individual versus group preference.

Interestingly, the training data used for evaluation by \cite{pan2013gbpr} consists of `standard' interaction datasets that do not contain explicit groups; rather, groups are sampled randomly among users who have consumed a particular item.
As such, Group BPR is perhaps best thought of as a different means of leveraging explicit and implicit signals in implicit feedback settings, rather than as a group method as such.

Experiments in \cite{pan2013} show that sampling pairwise preferences as in \Cref{eq:gbprSample} can improve performance over standard BPR on various benchmark datasets.
%\todo{boring}

\begin{figure}
\caption{When is side-information useful for recommendation?
\label{box:whenUseful}}
\begin{itemize}
\item Most of the settings we've considered in this chapter have essentially been forms of \emph{cold-start}. In other words, features features compensate for a lack of historical interaction data (from either the user or item side).
%, either as a form of regularizer, 
\item Features are \emph{unlikely} to be particularly useful in `warmer' settings: even if a feature (price, brand, genre) explains variance, high-variance dimensions will already be captured by latent terms (i.e.,~$\gamma_u$ and $\gamma_i$).
\item An exception to the above is features which are not \emph{static}. We study price variability in this chapter, and temporal dynamics more broadly in \Cref{chap:temporalSequential}. Latent terms cannot will struggle to capture this type of variability unless it is explicitly modeled.
\item Another important use of features is for model \emph{interpretability}: even features that yield a modest improvement in predictive performance may help us to understand the underlying dynamics of a particular problem better than we can from latent representations. We discuss such notions of interpretability in \Cref{chap:text}.
\end{itemize}
\end{figure}
%\todo{reference}

%\subsection{Price}
\section{Price Dynamics in Recommender Systems}
%\todo{Change below paragraphs to subsections}

In spite of the obvious impact price has on user decisions, there is surprisingly little work that seeks to incorporate price features into personalized predictive models.
%
Partly this owes to the lack of suitable datasets: the vast majority of the datasets we've study so far (concerned with movies, books, restaurants, etc.)~include few useful features from which to build a model of price.

Even in datasets that do include a price variable, it is not obvious how this variable should be incorporated in to the types of algorithms we've seen so far. Na\"ively one might think that price might be incorporated into (e.g.)~a factorization machine (\Cref{sec:?}) much like any other feature. While such a feature might help in cold-start settings, it is unlikely to improve predictive performance in general: to the extent that price explains significant variability in user preferences or item properties, it will already be captured by user or item latent factors. This form of ineffectiveness often comes as a surprise when implementing content-aware model: the features that explain the most variance (price, brand, genre, etc.)~are precisely those that latent-factor models already capture, and add little predictive capacity (see \Cref{box:whenUseful}).
%\todo{somewhat contradicts the claims of the first paper presented}
A notable exception to this is features that are not \emph{static}:
%We explore this scenario more broadly 
%However, 
while a simple feature like the price of an item may already be `baked in' to a latent factor representations, what our current models cannot tell us is how a user would react to a \emph{change} in price. As such, much of the research we'll explore below is concerned questions of price variability, and modeling the impact that a change in price will have on user preference.


%An Empirical Analysis of Algorithmic Pricing on Amazon Marketplace. Attempts to empirically detect algorithmic pricing among best-sellers on Amazon, and analyses their characteristics.
%https://mislove.org/publications/Amazon-WWW.pdf
%Not really relevant to personalizadion, though could cover in intro


%Developing a price-sensitive recommender system to improve accuracy and business performance of eCommerce applications (initial attempt to include price, doesn't seem super relevant)
Early 
%\todo{not early}
attempts at modeling price within the context of recommender systems explored the overall difficulty of
successfully incorporating price dynamics 
%into recommender systems.
\cite{umberto2015developing}.
%\todo{fold into intro sentence}



\subsection{Disentangling Prices and Preferences}

\cite{ge2011cost}
%\cite{ge2014cost}
considers price from the perspective of a user who wants recommendations that satisfy a \emph{budget constraint}.
%https://viscenter.uncc.edu/sites/viscenter.uncc.edu/files/CVC-UNCC-14-06.pdf
They consider this problem in the context of recommending `travel tours,' where a user has constraints in terms of time (the length of the vacation), and the amount they are able to spend. They note however, that while the length and price of an actual travel package is observed, a user's constraints may not be, and as such must be modeled (or estimated) based on their historical activities.

To achieve this type of price-aware recommendation, \cite{ge2011cost} consider a modification of a latent factor model that includes both a preference compatibility, and a price-compatibility term:
%\todo{no need to cite this all the time}
\begin{equation}
    f(u,i) = \underbrace{S(C_u, C_i)}_{\mathclap{\text{price compatibility}}} \cdot \overbrace{\gamma_u \cdot \gamma_i}^{\mathclap{\text{user preference}}}.
\end{equation}
$\gamma_u$ and $\gamma_i$ are user and item (travel tour)-related latent factors, as in \Cref{eq:?}. $C_u$ and $C_i$ are cost-related factors; the cost for the tour ($C_i$) is assumed to be an observed, two-dimensional vector, encoding both time and price; $C_u$ is assumed to be a corresponding latent compatibility encoding the user's price constraints; $S$ is then a compatibility function, such as the (negative) euclidean distance:
\begin{equation}
    S(C_u, C_i) = 1 - \| C_u - C_i \|_2^2.
    \label{eq:price1}
\end{equation}
A few improvements to this basic model are proposed (including in a follow-up paper \cite{ge2014cost}); for example, different training strategies are proposed based on different types of explicit and implicit feedback; and the user factor $C_u$ is carefully regularized (since e.g.~a trivial $\ell_2$ regularizer
%\todo{Do I use \ell_2 elsewhere?}
would center user price constraints around zero).

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/IPSim.pdf}
    \includegraphics[scale=0.75]{fig/NNSim.pdf}
    \caption{User vectors $\gamma_u$ in latent space, and candidate items to be recommended. Highly compatible items appear in the highlighted region under inner product (left) and Euclidean (right) compatibility models. \label{fig:InnerNNSim}}
\end{figure} 
%\todo{Move elsewhere and reference}

Experiments (on a proprietary dataset of historical travel tour interactions) show that the model outperforms variants that fail to consider price information.

%\todo{Maybe have paragraph here and raise levels of other subsections}
\cite{guo2017understanding} also builds a model to separately capture price and preference dynamics. Although the specific method is somewhat different from those we've discussed (a form of Poisson Factorization, see e.g.~\cite{gopalan2013scalable}), this approach has a common goal with \cite{ge2011cost} of separating price and preference concerns. In essence, latent item properties $\gamma_i$ are responsible for estimating ratings via $\gamma^{(\text{rating})}_u \cdot \gamma_i$ and also price compatibility via $\gamma^{(\text{price})} \cdot \gamma_i$; compatible items are then those that satisfy both of these concerns.
%\todo{Rough paragraph, doesn't really fit anywhere}
%\todo{Add moral of the story about the point of disentangling price / preference concerns}

\subsection{Estimating Willing-to-Pay Prices within Sessions}
%\todo{This really belongs in the above section since it's basically the same idea}

\cite{hu2018learning} also considered the effect that price has on users' purchasing decisions, but did so at the level of individual browsing sessions. That is, the sequence of products a user browses might provide some clue as to their purchase intent or their `willing-to-pay' amount, e.g.~if they are comparison shopping among items within a certain price range.

Like \cite{ge2011cost}, the basic model of \cite{hu2018learning} extends a latent-factor
%\todo{latent-factor or latent factor?}
model to incorporate a price-compatibility term:
\begin{equation}
    f(u,i) = \gamma_u \cdot \gamma_i + \alpha_u C(u, p_i).
\end{equation}
Here $C(u, p_i)$ encodes the compatibility between the user $u$ and the price $p_i$ of the item $i$, and $\alpha_u$ is a personalized measure of user $u$'s sensitivity toward this term.

A trivial price-compatibility term might take the form
\begin{equation}
    C(u, p_i) = \exp\left( -\omega (b_u - p_i)^2 \right)
\end{equation}
where $p_i$ is the price of the item and $b_u$ is a latent estimate of the user's budget; this function is essentially a variant of \Cref{eq:price1}.

To incorporate session dynamics, \cite{hu2018learning} considers a price-compatibility term based on a feature vector $\rho_i$, which is a one-hot encoding representing the price \emph{quantile} of the price $p_i$ compared to the prices of previously-viewed items in the session;\footnote{For example, $\rho_i = [0,0,0,1]$ would indicate that the price $p_i$ was among the top 25\% of browsed prices.} then the price-compatibility is merely $C(u, p_i) = \theta \cdot \rho_i$. This is further extended by using a \emph{mixture model}, which essentially says that there could be different parameters $\theta_g$ for different (latent) users groups:
\begin{equation}
    C(u, p_i) = \sum_g \underbrace{\frac{e^{\psi_{u,g}}}{\sum_{g'}e^{\psi_{u,g'}}}}_{\mathclap{\text{extent to which $u$ belongs to group $g$}}} \overbrace{\theta_g \cdot \rho_i}^{\mathclap{\text{price-compatibility model for group $g$}}}.
\end{equation}

What \cite{hu2018learning} ultimately find is that there are several different classes of user (based on latent group membership $\psi_{u,g}$): some tend to gradually browse toward more expensive items, others gradually browse toward cheaper items, 

%Cost-Aware Collaborative Filtering for Travel Tour Recommendations

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Comparison of price-aware recommendation techniques.}
\label{tab:price}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{lXX}
% \hrule\\
\textbf{Reference} & \textbf{Dataset} & \textbf{Description}\\
\cite{ge2011cost} & (Proprietary) data of travel-tour purchases & Incorporates price and time constraints into travel-tour recommendations\\
\cite{guo2017understanding} & Various \emph{Amazon} categories & Disentangles interactions in terms of preferences versus price compatibility.\\
\cite{wan2017modeling} & Purchases from Seattle grocery stores & Estimates how purchase decisions (item choice, quantity, etc.)~are affected by price fluctuations.\\
\cite{hu2018learning} & Purchase and browse data from \emph{Etsy} & Forecasts a user's target purchase price from a sequence of browsed items.\\
% \hrule
\end{tabularx}
\end{minipage}
\end{table}
%\todo{table formatting}

\subsection{Price Sensitivity and Price Elasticity}

Notions such as \emph{price sensitivity}, and \emph{price elasticity} (defined as the change in purchase quantity given a change in price) are well understood in economics and marketing \cite{case2007principles}. Understanding such factors can help to guide custom marketing and promotion strategies \cite{zhang2004customizing,zhang2009effectiveness}.

However they are less well understood in terms of their effectiveness in a \emph{predictive} setting, i.e.,~in terms of how price should be used to understand and forecast user actions (or to make recommendations).

Part of the reason that price has received relatively little attention (at least in academic literature) is presumably the lack of useful available data; even when price is observed, it is confounded by numerous other factors, such as brand or manifest aspects of a product; moreover even when price data is available, on rarely has \emph{historical} data on price that allows for measurement of the impact of a \emph{change} in price.

\cite{wan2017modeling} studied price in the context of grocery recommendation. Their research was mostly based on real transaction data from a physical grocery store (in Seattle), though was also validated based public data from \emph{dunnhumby}. Both datasets contain price measurements, and critically measurements of price \emph{variation} over time. As such the main questions center around the extent to which purchase decisions are affected by changes in price.

In the context of grocery shopping, potential questions include:
\begin{itemize}
    \item Will a reduction in price cause a user to buy a \emph{category} of product they otherwise wouldn't have (e.g.~would they buy milk at a discount if it wasn't on their shopping list)?
    \item Will a reduction in price cause users to buy a \emph{specific item} that they otherwise wouldn't have (e.g.~would they buy a different brand of milk because of a discount)?
    \item Will a reduction in price cause users to buy a \emph{larger quantity} of an item than they otherwise would have?
\end{itemize}
To study these questions, prediction is broken down into three subsequent choices:
\begin{multline}
    p_u(\text{buy $q$ units of an item $i$ from category $c$}) = \\
    \begin{tabular}{l}
      $\hphantom{\times} p^{(\text{category})}_u(\text{buy a product from category $c$})$ \\
      $\times p^{(\text{item})}_u(\text{buy product $i$\ } | \text{\ buying from category $c$})$\\
      $\times p^{(\text{quantity})}_u(\text{buy $q$ units\ } | \text{\ buying item $i$})$.
    \end{tabular}
\end{multline}

Each of these three prediction tasks (category, item, and quantity prediction) is based on a predictor $f(u,c,t)$, $f(u,i,t)$, and $f(u,q,t|i)$; the underlying method behind each is a latent-factor model, as in \Cref{eq:latentSomethnig}, including additional features associated with the time (e.g.~what day of the week the trip occurs on). Each is passed through a different activation function, for example quantity prediction is modeled via a \emph{Poisson} function:
\begin{equation}
    p(\text{quantity} = q | \text{buying item $i$}) = \frac{f(u,q,t|i)^{q-1}\exp(-f(u,q,t|i))}{(q-1)!}.
\end{equation}

Next, the change in purchase probability due to price is captured (for each of the three models) using a simple feature encoding the price at a particular point in time. Specifically (e.g.~for quantity)
\begin{equation}
    f'(u,q,t|i) = f(u,q,t|i) + \beta_{u,q} \log P_i(t),
\end{equation}
where $P_i(t)$ is the price of the item at time $t$. $\beta_{u,q}$ is then a bias term which encodes the price sensitivity, i.e., extent to which a particular user $u$, when purchasing $q$ units of an item, will react to changes in price; a negative value of $\beta_{u,q}$ would indicate that a user is less likely to purchase a particular quantity given a price increase. All parameters are learned by training on purchase data, using a BPR-like training scheme.

%\todo{Talk about WARP / WARP 2.0 loss (not here, I just rememebered)}

\emph{Price-elasticity} now reflects how much preference changes given a change in price, e.g.~for a particular item $i$:
\begin{equation}
    e_{i,u}(t) = \frac{df'(u,i,t)}{f(u,i,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx (1 - f'(u,i,t))\beta_{u,i}(t).
\end{equation}
%\todo_{Redo this, fix notation, \beta is the output of some model, not a linear coefficient}
A related notion, \emph{cross-elasticity}, measures the extent to which a change in $i$'s price will change a user's compatibility toward (e.g.~probability of purchasing) a \emph{different} product $j$:
\begin{equation}
    e_{i,j,u}(t) = \frac{df'(u,j,t)}{f(u,j,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx -f(u,i,t)\beta_{u,i}(t).
\end{equation}
%\todo_{CHECK!}
% Compare to equality in https://mengtingwan.github.io/paper/www17_mwan.pdf
% (after eq 15)
%a(\sum_(j neq a) j)b_i  =   \sum_{j neq a} ja b_i

Note that the above equations (price-elasticity and cross-elasticity) are \emph{measurements} after the model has been trained.
%\todo_{Crappy sentence}
The main finding of the model is that price-elasticity applies mostly to product choice, but not to category choice or quantity.
%\todo_{a few other findings}

%\todo{Write f's as PROBABILITIES}


%Others:
%Shopper: A probabilistic model of consumer choice with substitutes and complements (Ruiz, Athey, and Blei): focused on counterfactual queries about changes in price, and includes some study of complementary items. Somewhat similar to above, also for groceries.
%\cite{ruiz2020shopper}
\cite{ruiz2020shopper} develops a somewhat similar model of consumer choice, also in a setting of grocery purchases. Like the above model, \cite{ruiz2020shopper} attempts to disentangle the various effects of item popularity, user preference, and price dynamics, though also includes additional terms involving seasonal effects. The main goal of the paper is to answer `counterfactual' queries about price (i.e.,~what would the user have done if the price had been different?).
%~in the presence of substitutable and complementary items.
%They do so by analyzing 
By modeling how users will react to changes in price, they argue that the model is also able to detect interactions between products, namely in terms of which items are likely to be substitutable and complementary.
%https://arxiv.org/pdf/1711.03560.pdf
%\todo{Fine, but no mention of why I'm not talking about this paper more}
%
%
%\subsection{Optimizing Promotion Strategies}
%\todo{finish, could include papers below}
%
%

Finally, we mention attempts to use 
similar ideas within the context of 
%designing customized promotions
dynamic pricing.
\cite{jiang2015redesigning} seek to combine ideas from ideas from from pricing and recommendation in order to design optimal (i.e.,~profit-maximizing) promotion strategies.
%Redesigning promotion strategy for E-commerce Competitiveness through pricing and recommendation (also somewhat related to price sensitivity, attempts to use such ideas to design profit-maximizing promotions.
%
%An Empirical Analysis of Algorithmic Pricing on Amazon Marketplace. Attempts to empirically detect algorithmic pricing among best-sellers on Amazon, and analyses their characteristics.
Conversely,
\cite{chen2016empirical} analyzes the characteristics of sellers on \emph{Amazon}, in order to automatically detect the presence of algorithmic pricing.
%\todo{not the best, could drop. Otherwise check that it's accurate}


\section{Other Contextual Features in Recommendation}
%\todo{probably make subsection and reduce these to paragraphs}

\subsection{Music and Audio}

Improving Content-based and Hybrid Music
Recommendation using Deep Learning
\cite{wang2014improving}
%https://dl.acm.org/doi/pdf/10.1145/2647868.2654940?casa_token=UtHgvyXPnEIAAAAA:wVMpUDix7muDjz3gpw_pq1PIxExffT9gjXxZBf8oymBo8Dydpj6qBfvO1xjD7CNoqKTz3PbediLn4A
%Quite straightforward  - item embedding is learned from DNN

\cite{wang2014improving} notes the difficulty of directly using high-dimensional audio features (e.g.~based on spectrogram-based representations of audio) within a traditional feature-based recommender system. The solution they propose is to use a neural network-based representation (essentially a multi-layer perceptron) to learn embeddings of songs that are useful for recommendation, i.e.,:
\begin{equation}
    \gamma_i = \mathit{MLP}(x_i)
\end{equation}

Deep content-based music recommendation
\cite{van2013deep}
%https://papers.nips.cc/paper/2013/file/b3ba8f1bee1238a2f37603d90b58898d-Paper.pdf

Million song dataset
%\cite{bertin2011million}
\cite{mcfee2012million}

\subsection{Recommendation in Location-Based Networks}
%\todo{And something else...?}



%https://link.springer.com/content/pdf/10.1007/s10707-014-0220-8.pdf

%\todo{Refer to this section from location section in sequential modeling}

Several attempts have been made to incorporate geographical features into recommender systems. Actions are often guided by geographical constraints, whether due to a user operating within a certain geographical region, or due to sequential actions being highly localized.
%\todo{bad opening sentence}
\cite{bao2015recommendations}

%\todo{finish}

\subsection{Temporal, Textual, and Visual Features}

So far we have covered 
content in recommender sytems, 
including features ranging from price, geography, social signals, and audio.
%Segue into next chapter
%\done{Pretty useless paragraph}
In the following chapters we'll revisit content-aware recommendation techniques using features based on temporal and sequential dynamics (\Cref{chap:temporalSequential}, text (\Cref{chap:text}), and images (\Cref{chap:visual}). These modalities require special attention, primarily to deal with the complex semantics of the data and high-dimensional signals involved. Exploring how to build personalized models based on these complex modalities of data is one of the main themes explored throughout the remainder of this book.

\section{Exercises}

\begin{exercises}
 \item %Fit a factorization machine with various features
 In \Cref{sec:factorization?} we introduced the Factorization Machine  as a general-purpose technique for incorporating features into recommender systems. In this exercise we'll incorporate a few features into a factorization machine to measure the extent to which they improve recommendation performance. You could use any dataset for this exercise, so long as it includes a few features. For example, using our beer dataset (as in \Cref{?}) we might include features such as:
 \begin{itemize}
     \item Simple numerical features such as the ABV of the beer or the age of the user.
     \item The category of the item (one-hot encoded).
     \item The timestamp; this will require some care to encode, e.g.~you might use an encoding of the season or day of week.
 \end{itemize}
 %\todo{Why don't bullet points look the same as in other exercises?}
 Using a few of these features (or similar features on another dataset), measure the extent to which performance is improved by adding features to the recommender.
 %\todo{reword a little}
 \label{ex:FM1}
 \item In \Cref{sec:?} we discussed the potential value of incorporating features (side information) into recommender systems, and argued that features may be most informative in \emph{cold-start} scenarios. To assess this, conduct the following experiment (using your model from \Cref{ex:FM1}):
 \begin{itemize}
     \item For each user (or item) in the test set, count how many times that user appeared in the training set.
     \item Plot the testing performance (y) of your model as a function of how many times that user appeared in the training set (x).
     \item Generate the same plot both with and without additional features in your factorization machine.
 \end{itemize}
 Naturally, our expectation is that performance will improve as users (or items) are less `cold' (i.e.,~more training interactions). However, we expect the performance degredation to me more mild when features can compensate for the lack of interactions.
 \item Before implementing a socially-aware recommender, here we'll test the hypothesis that friends actually interact with more (or less) similar  items compared to randomly chosen sets of users. This could be measured in various ways, e.g.:
 \begin{itemize}
     \item Compute the average Jaccard (or cosine, etc.)~similarity between randomly chosen pairs of users, versus randomly chosen pairs of friends.
 \end{itemize}
 \item In \Cref{sec:?} we explored a few ways to incorporate social signals into one-class recommender systems (\Cref{sec:?}).
 %\todo{Do I call them "one class" or something else?}
 For the most part,
 %\todo{is it most or all of them?}
 these techniques amounted to ways of sampling negative feedback instances, e.g.~we might be more (\Cref{eq:?}) or less (\Cref{eq:?}) to interact with negative instances that our friends have already interacted with. Using a dataset that contains social interactions (e.g.~\ref{?}, as in \Cref{sec:?}) experiment with these sampling strategies and determine which (if any) lead to improved performance over a traditional BPR implementation.
 %\todo{BPR defined or do I always write it out? Should I here?}
 \item Incorporate price into some dataset (amazon?) in a few ways
\end{exercises}



\stepcounter{projectCounter}
\subsection{Project \theprojectCounter: Cold-Start Item Recommendations on \emph{Amazon}}
%\todo{Replace with a specific category}

As we argued in \Cref{sec:?},
%\todo{did we?}
one of the main reasons to incorporate features into personalized recommendation approaches is to improve their performance in \emph{cold-start} settings. Here, we'll look at cold-start recommendation problems using data from \emph{Amazon}, following data from (e.g.)~\cite{?}. We select this dataset as it includes various types of item metadata (prices, brands, categories, etc.)~that can potentially be useful in cold-start settings, though this project could be completed using any dataset that includes user or item metadata.

We'll build our system for cold-start recommendation via the following steps:

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/rmseNtrain}
\caption{Performance (MSE) as a function of user coolness (defined as the number of times that user was seen during training). Goodrdeas `Graphic Novels' data.}
%\todo{style}
%\todo{refer to figure}
\end{SCfigure}

\begin{enumerate}
    \item Start by implementing a `vanilla' factorization machine to solve the prediction problem, i.e.,~without incorporating any side-information.
    Note that this problem could be cast either as one of rating prediction (as in \Cref{sec:?}) or as purchase prediction (as in \Cref{sec:?}).
    %\todo{Same text as next project}
    \item To build models for cold-start recommendation, it is useful to develop some evaluation metrics specifically for the purpose of evaluating cold-start (and `cool-start') performance. To do so, try plotting the performance on the test set \emph{as a function of the number of times the item appears in the training set}. Our hypothesis when incorporating side-information into recommendations is that performance improvements will be largest for the coldest items, and less useful for items with longer interaction histories. We'll use this same type of plot to compare models in the following.
    %\todo{Make plot of this, but for a normal model}
    \item Several features could potentially be useful in cold-start scenarios. Consider how to encode the following: (a) the brand of the item; (b) the category (or categories) the item belongs to; and (c) the price of the item. You might also consider more complex features based on (e.g.)~the text of the description or temporal information, though we'll revisit these topics in \Cref{chap:temporalSequential,chap:text}.
\end{enumerate}
For each cold-start feature you include, compare
%\todo{Needs something else to finish the project}





\chapter{Temporal and Sequential Models}
\label{chap:temporalSequential}

%So far, we have
Throughout \Cref{chap:recsys,chap:?} we developed gradually more refined arguments about the role that features (or `side-information') play when modeling user interaction data. Our initial argument (started in \Cref{?}) was that latent user and item representations are sufficient to capture complex preference dynamics, and whatever features are most predictive (i.e.,~explain the most variability) of interactions will automatically emerge via our latent representations.

Later, (in \Cref{sec:?}) we refined this argument, arguing that side-information \emph{can} be useful, especially in settings where there is a paucity of interaction data from which to learn high-dimensional latent representations (i.e.,~the \emph{cold-start} setting).

However in both of the above cases we still assumed that our model of users and items was \emph{stationary}. In practice, preferences and interactions can be non-stationary for a variety of reasons. For instance, no matter how many interactions we observe, and how many latent factors we fit, a model such as the one we developed in \Cref{sec:?} cannot tell us that a user might only buy swimsuits in summer, or that they are unlikely to watch the third film in a series until they have already seen the second.

In this chapter we explore techniques to build personalized models around user behavior that has \emph{temporal and sequential dependencies}. Most straightforwardly, we might simply treat temporal and sequential information as additional features that can be modeled, e.g.~via the framework of a Factorization Machine as we developed in \Cref{chap:?} (and indeed we will explore temporal models based on this type of approach). However as we will see in this chapter, a variety of complex and subtle temporal dynamics could be at play, for example:
\begin{itemize}
    \item Dynamics could apply to users (e.g.~users may grow out of certain movies), items (e.g.~a movie's special effects may become dated over time), or the \emph{zeitgeist} of the entire community may shift over time.
    \item Temporal dynamics can exist at multiple scales, e.g.~when modeling heart-rate data (\Cref{sec:caseStudyHR}), dynamics can be short-term (e.g.~a user running), medium-term (e.g.~a user getting tired) or long-term (e.g.~a user becoming more fit).
    \item In addition to short- and long-term dynamics, dynamics can also be periodic (e.g.~weekly or seasonal trends), or prone to bursts and outliers (e.g.~purchases around a major holiday).
\end{itemize}

Our main focus in this chapter is to explore a wide variety of (personalized) models for temporally-evolving data. As we'll see, dynamics such as those above require carefully-designed models (rather than simply including a temporal feature in a general-purpose model). As such, we focus on understanding the overall process and design considerations when building personalized models with temporal dynamics.

\section{Introduction to Regression with Time Series}
%\todo{This would be better in regression chapter, but probably only do that if regression + classification become two separate chapters}
%\todo{Note that actually sequential recommender systmes *are* autoregressive, so this is a perfectely reasonably place to discuss autoregression}

Before investigating more sophisticated models of temporal and sequential data, it is instructive to consider how much progress can be made with techniques we have already developed.

To do so, we'll consider developing predictors that estimate the next value in a sequence (or the next several values). In the simplest case, we are given a sequence of observations $y = \lbrace y_1 \ldots y_n \rbrace$, from which one would like to predict the next value ($y_{n+1}$). For example, one might want to estimate website traffic on the basis of historical traffic patterns.

To solve such a problem using a regression approach (or a classification approach for a binary outcome), we can imagine constructing features based on the previous observations ($\lbrace y_1 \ldots y_n \rbrace$) in order to predict the next one. Presumably, once we observe the true value of $y_{n+1}$, we would like to predict $y_{n+2}$, and so forth. Note that this blurs the line somewhat between `features' and `labels,' since the label for one prediction becomes a feature for the next.

This procedure is known as \emph{auto-regression} (referring to the fact that we are regressing on the same data that was used for prediction). As usual, we are typically interested in defining our regressor (or classifier) such that it minimizes some error between the predictions and the true values. That is, we want to define a predictor $f(y_1 \ldots y_n)$ that estimates the next value ($y_{n+1}$) in the sequence so as to minimize (e.g.)~a Mean Squared Error:
%\todo{Capitalization?}
\begin{equation}
    \frac{1}{n} \sum_{i = 1}^{n} (f(y_1 \ldots y_i) - y_{i+1})^2.
\end{equation}

Trivially, we might imagine several na\"ive techniques for estimating the next value in a sequence, e.g.~we could predict the next value as a weighted sum of previous values:
\begin{eqnarray}
    \text{moving average:}\quad f(y_1 \ldots y_n) & = & \frac{1}{K} \sum_{k = 0}^{K - 1} y_{n - k} \label{eq:movingAvarege}\\
    \text{weighted moving average:}\quad f(y_1 \ldots y_n) & = & \frac{\sum_{k = 0}^{K - 1} y_{n - k}}{\sum_{k=1}^K k}. \label{eq:weightedMovingAverage}
\end{eqnarray}
Although simple, one can imagine how these averages could potentially be better predictors than always predicting the next value to be equal to the previous (for example, see \Cref{ex:autoRegression1}). These types of trivial predictors can also be used to plot trends in noisy data (\Cref{fig:slidingWindows}); averages over larger intervals (i.e.,~larger values of $K$) will produce smoother summaries of the data. Such averages can be efficiently computed for successive values via a dynamic programming solution; code to generate the plots in \Cref{fig:slidingWindows} is presented below:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
xSum = sum(x[:wSize]) # Given data x and y to plot, and a window size wSize
ySum = sum(y[:wSize]) # Sum of first wSize values
xSliding = []
ySliding = []

for i in range(wSize,len(x)-1):
    xSum += x[i] - x[i-wSize] # Strip off oldest value and add newest one
    ySum += y[i] - y[i-wSize]
    xSliding.append(xSum / wSize)
    ySliding.append(ySum / wSize)
\end{lstlisting}
\end{minipage}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/slidingWindows.pdf}
\caption{Moving-average plots of $\simeq$1 week of \emph{Goodreads}
%\todo{italicize?}
Fantasy novel ratings. Although not particularly effective as predictors, moving averages can be used to plot data so as to summarize overall trends. \label{fig:slidingWindows}}
\end{SCfigure}
%\todo{Probably add code here}

%\todo{`Autoregression' is never defined. At least name this a paragraph or subsection}

The two trivial strategies above are heuristics for predicting the next value, capturing the intuition that it should be similar to recently observed values (\Cref{eq:movingAvarege}), and possibly that more recent values should be more predictive than less recent ones (\Cref{eq:weightedMovingAverage}). However, a better strategy might be to \emph{learn} which of the recent values are the most predictive, i.e.,
\begin{equation}
    f(y_i \ldots y_n) = \sum_{k=0}^{K - 1} \theta_k y_{n - k}.
    \label{eq:autoRegression}
\end{equation}
The values $\theta_k$ now determine which of the previous $k$ values are predictive. For example, in periodic data (e.g.~network traffic, seasonal purchases), the most predictive values may include recent observations, observations from 7 days ago, observations from the same day in the previous month, etc.

%\todo{Include example from some actual dataset? Could use e.g. Bay Area Bike Share, though dataset seems not to be public}

For example, training a simple autoregressive model on a dataset of hourly measurements of bike rentals in the Bay Area\footnote{Each observation $y_h$ is a measurement of how many trips were taken during hour $h$.} yields the following model:
%\todo{Fix equation alignment}
\begin{equation}
%0.47105067, -0.28432565, 0.10555845,  0.01445134, -0.02131945, 0.17501989,  0.539696351
    y_{n} = \begin{tabular}{l}
        0.471 $\times y_{n-1}$\\
      - 0.284 $\times y_{n-2}$\\
      + 0.106 $\times y_{n-3}$\\
      + 0.014 $\times y_{n-4}$\\
      - 0.021 $\times y_{n-5}$\\
      + 0.175 $\times y_{n-24}$\\
      + 0.540 $\times y_{n-24\times 7}$
      \end{tabular}
\end{equation}
Here we see that the two most predictive observations are those from the previous hour ($y_{n-1}$) and from exactly one week ago ($y_{n - 24\times 7}$). Observe that we didn't include \emph{every} previous observation up to $24\times 7$ hours ago as a feature, rather we only included those previous observations that we expected to be predictive.

Note that although the solution in \Cref{eq:autoRegression} uses \emph{only} previous values in the sequence, one can of course include other features associated with the current timepoint or previous values, just like in a normal regression model.

%\todo{Check indentation level for all lstlistings}

Although a simple approach to regression and classification of time series data, the basic idea behind auto-regression (i.e.,~to use previous observations to predict future values in a sequence) will reappear in many of the more complex models we develop, especially as we develop sequential models in \Cref{sec:sequential?}.

\section{Temporal Dynamics in Recommender Systems}
\label{sec:temporalRec}

Several attempts have been made to improve recommendations by incorporating temporal dynamics. There are countless reasons why preferences, purchases, or interactions may change over time, or more simply why knowing the current timestamp may help us to more accurately predict the next interaction.
Consider for example the following scenarios which could cause changes in movie ratings or interactions over time:
\begin{enumerate}
    \item Users may give higher ratings to older movies, e.g.~due to feelings of `nostalgia'.
    \label{temporalItem1}
    \item Alternately, ratings of older movies may represent a biased sample of items that users had explicitly searched for (versus newer items which a user selected due to their being surfaced on a landing page).
    \label{temporalItem2}
    \item A mundane change to a user interface, such as modifying the tool-tip text associated with a certain rating, may alter the rating distribution.
    \label{temporalItem10}
    \item Users who favor special effects may give lower ratings as a movie's special effects become dated.
    \label{temporalItem3}
    \item A family member may borrow a user's account, and temporarily consume movies quite different from the account's typical preferences. 
    \label{temporalItem4}
    \item Users may binge-watch a series, dominating their interaction patterns for a short period.
    \label{temporalItem5}
    \item Action blockbusters may be more favored during summer (or Christmas movies during Christmas).
    \label{temporalItem6}
    \item Users may want to consume (or avoid) content very similar to what they have previously interacted with.
    \label{temporalItem7}
    \item Users may gradually develop an appreciation for certain characteristics of a movie as they consume more content from that genre.
    \label{temporalItem8}
    \item Users may be anchored by external forces, i.e.,~the \emph{zeitgeist} of what is currently popular in their community.
    \label{temporalItem9}
\end{enumerate}
%INCLUDE ITEMS BELOW
% Changes in the interface
% • People give higher ratings to older  movies (or, people
% who watch older movies are a biased sample)
% • The community’s preferences gradually change over time
% • My girlfriend starts using my Netflix account one day
% • I binge watch all 144 episodes of buffy one week and
% then revert to my normal behavior
% • I become a “connoisseur” of a certain type of movie
% • Anchoring, public perception, seasonal effects, etc.
The above dynamics are quite varied in their sources and scale: Effects~\ref{temporalItem1} and \ref{temporalItem3} are gradual and long-term; \ref{temporalItem1} and \ref{temporalItem10} owe to vageries of a changing user interface; \ref{temporalItem4} and \ref{temporalItem5} are `bursty' or short-term; \ref{temporalItem6} is seasonal; \ref{temporalItem7} is sequential; \ref{temporalItem8} owes to user growth; and \ref{temporalItem9} due to a changing community. One can imagine many other sources, especially in different settings subject to social dynamics, price variability, fashion, etc.
%
As we will see, understanding such dynamics is often key to making successful recommendations. 
%Some may seem mundane or trivial: e.g.~a change in UI or 
Some may scarcely seem like `temporal dynamics' at all: e.g.~a change to the user interface has little to do with user or item evolution. Nevertheless, we'll argue that modeling even these trivial or mundane dynamics proves critical in order to disentangle them from the `real' real personalization dynamics in the data.

\paragraph{Methods for Temporal Recommendation}
Methods for temporal recommendation fall broadly into two classes. The first make use of the actual \emph{timestamps} of events. Each interaction $(u,i)$ is augmented with a timestamp $(u,i,t)$, and the goal is to understand how ratings $r_{u,i,t}$ vary over time. That is, our goal is to extend models such as the one in \Cref{sec:latentFactor} so that parameters vary as a function of time, e.g.
\begin{equation}
    r_{u,i,t} = \alpha(t) + \beta_u(t) + \beta_i(t) + \gamma_u(t) \cdot \gamma_i(t).
    \label{eq:latentFactorTime}
\end{equation}
Modeling temporal dynamics in this way is effective at capturing long-term shifts of community preferences over time; short-term or `bursty' dynamics, such as purchase patterns being affected by external events, or periodic events, such as purchases being higher at a particular time of day, day of week, or season.

A second class of methods discards the specific timestamps, but preserves only the \emph{sequence} (or order) of events. Thus the goal is generally to predict the next action as a function of the previous one, i.e.,
\begin{equation}
    p(\text{user $u$ interacts with item $i$} | \text{they previously interacted with item $j$}).
    \label{eq:sequenceModelExamp}
\end{equation}
This type of model makes the assumption that the important temporal information is captured by the \emph{context} provided by the most recent event. This is useful in highly-contextual settings, such as predicting the next song a user will listen to, or other items they will place in their basket, etc. In such settings knowing the most recent actions (or most recent few actions) is more informative than knowing the specific timestamp.

These two classes of model are quite orthogonal, both in terms of the settings where they are effective, as well as the techniques involved. Below we'll explore both settings via several case-studies. First we'll explore parametric temporal models (as in \Cref{eq:latentFactorTime}) through the example of the Netflix prize, where temporal dynamics 
%\todo{italics}
are carefully modeled to capture a wide variety of application-specific dynamics. We explore sequential models (as in \Cref{eq:sequenceModelExamp} in \Cref{sec:markovCaseStudies} via several examples, starting with online shopping scenarios; these settings tend to be highly contextual, where the context of recent activities is often more informative than long-term historical activities. 

Later, 
we'll introduce \emph{recurrent networks} as general-purpose models of sequence data (\Cref{sec:?}), and as a potential approach to capture complex dynamics in evolving interaction sequences. Such models have the potential to overcome the limitations of traditional sequential models by learning complex semantics that persist over many steps. Such models arguably represent the current state-of-the-art for general purpose recommendation. 
%Our discussion on recurrent networks leads us toward \Cref{chap:text}, where such 
%\todo{In this chapter or not?}

%\todo{Wrote above late, see if it fits}

%\todo{Borrowed paragraphs}
%Some early works that deal with temporal dynamics do so in
%terms of concept drift 
%[23, 37, 39]. 
%Such models are able to account
%for short-term temporal effects (‘noise’), and long-term changes
%in user behavior (‘drift’), for example due to the presence of new
%products within a community.

%Sophisticated models of such temporal dynamics proved critical
%in obtaining state-of-the-art performance on the Netflix challenge
%[3], most famously in [20]. As discussed in [20], few previous
%works had dealt with temporal dynamics, other than a few notable
%exceptions [2, 10, 35]. Around the same time, ‘adaptive neighborhood’ models were proposed [24], that address the problem of
%iteratively training recommender systems whose parameters ought
%to change over time.

% Better performance may be obtained by modeling large-scale
% global changes at the level of entire communities [41], or by developing separate models for short term changes (e.g. due to external
% events), and long-term trends [40].
% Other works that study temporal dynamics at the level of products and communities include [29], where the authors studied how
% existing ratings within communities may influence new users, and
% how community dynamics evolve; and [13], who studied how users
% are influenced by previous ratings of the same product

%\todo{Introduce why we're talking about this}

\subsection[Case Study: Temporal Recommendation and the Netflix Prize]
{Case Study: Temporal Recommendation and the Netflix Prize \cite{koren2009collaborative}}
\label{sec:caseStudyNetflixTemporal}
%\todo{Should include citations in section headings?}

%\todo{Several typos below}

Careful modeling of temporal dynamics was one of the key features characterizing the strongest solutions to the Netflix Prize. Several of the ideas that proved effective for modeling temporal dynamics on Netflix specifically are covered in \cite{koren2009collaborative} (``Collaborative Filtering with Temporal Dynamics''), which we summarize here.

As a motivating example, consider the two plots shown in \Cref{fig:netflixTemporal}.\footnote{These two plots are based on similar figures from \cite{koren2009collaborative}; the purported trends are somewhat more pronounced in the original paper, possible due to differences in sampling schemes.} At top, we see ratings over time averaged across weekly bins. We see, several points where ratings appear to increase suddenly, followed by plateaus of relatively stable ratings; for example at around week 210, average ratings appear to jump from around 3.5 stars to 3.6 stars.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/netflix1.pdf}\\
    \includegraphics[scale=0.75]{fig/netflix2.pdf}
    \caption{Temporal dynamics on Netflix. The top plot shows ratings averaged across each week over the lifetime of the dataset; the bottom plot shows how ratings change for newly-introduced movies, showing that ratings gradually increase during the first few weeks the movie is on Netflix. These plots reveal a combination of sudden and gradual trends in movie ratings over time.     \label{fig:netflixTemporal}}
\end{figure}
%\todo{Plots don't fit; should be shrunk width-wise}

Such long-term, population-level changes could owe to several explanations. Changes in rating patterns could be due to changing user base, or from certain movies being added to Netflix; such changes could even be due to world events exogenous to Netflix. Or, the change could owe to a factor as simple as a change in Netflix's User Interface (UI), causing users to rate movies differently.

The bottom plot in \Cref{fig:netflixTemporal} shows another temporal trend, demonstrating that individual movies recieve higher ratings the longer they have been available on Netflix. Again, such trends could be due to a variety of factors: users may be favorably biased toward older movies (e.g.~by nostalgia); or again it could be a function of the UI: a user who specifically sought out an older movie may view it more favorably than a user who had been recommended the same movie from the front page.

Whatever the underlying cause of these trends, they account for significant variability in the observed ratings. And as such, we should model these dynamics to predict ratings more accurately. Na\"ively, one might simply discard (e.g.)~older data that does not correspond to the predominant rating trends. But, a more effective model would attempt to account for the differences between newer and older data.

To model the kinds of long term trends captured in \Cref{fig:netflixTemporal}, \cite{koren2009collaborative} first focuses only on temporally evolving bias terms, i.e.,
\begin{equation}
    b_{u,i}(t) = \alpha + \beta_u(t) + \beta_i(t).
\end{equation}
Starting with item biases, one can capture long-term, gradual variation simply by having different bias terms for different periods, i.e.,
\begin{equation}
    \beta_i(t) = \beta_i + \beta_{i,\mathrm{Bin}(t)}.
    \label{eq:bins30}
\end{equation}
In \cite{koren2009collaborative} the authors suggest $\sim 30$ bins corresponding to about 10 weeks each in the case of the Netflix data.

This basic idea of separating bias terms into bins could likewise be applied to capture periodic trends, much like we encoded periodic terms in \Cref{sec:temporalFeatures}. Here the bias term would again take the form of several bins:
\begin{equation}
    \beta_i(t) = \beta_i + \beta_{i,\mathrm{Bin}(t)} + \beta_{i,\mathrm{period}(t)},
\end{equation}
where $\mathrm{period}(t)$ could represent periodic effects at the level of different days of the week, or months of the year (etc.).

The above ideas are effective for modeling long-term and periodic dynamics, but are quite expensive: e.g.~the model in \Cref{eq:bins30} requires an additional 30 parameters \emph{per item}; this is affordable for items on Netflix since the average item has over 5,000 ratings (100 million ratings of 17,770 titles). However this would likely not be possible for users, as the average user has only around 200 ratings.
% 480k users
Thus one needs a way of parameterizing user temporal dynamics that is considerably cheaper (i.e.,~involves fewer parameters).

A solution suggested in \cite{koren2009collaborative} is to use an `expressive deviation' term for each user:
\begin{equation}
    \mathrm{dev}_u(t) = \underbrace{\mathrm{sign}(t - t_u)}_{\mathclap{\text{before $(-1)$ or after $(1)$ the mean date}}} \cdot |t - t_u |^x.
    \label{eq:expressiveDev}
\end{equation}
The term $t_u$ represents the mean date amongst a particular user $u$'s ratings, so that the term $(t - t_u)$ represents whether a particular point in time $t$ is before or after the midpoint of the user's rating lifetime.
%
The expressive deviation term is depicted in \Cref{fig:expressiveDev}; the exponent of 0.4 was found to work well on Netflix data. 
%\todone{sentence}
The deviation term augments the user bias term via a user-specific scaling term $\alpha_u$, essentially controlling how strongly the deviation term applies to a specific user:
\begin{equation}
    \beta_u(t) = \beta_u + \alpha_u \cdot \text{dev}_u(t),
    \label{eq:temporalBeta}
\end{equation}
e.g.~a negative value of $\alpha_u$ would mean the user's ratings trend down over time, whereas a value $\alpha_u \simeq 0$ would mean that the user's overall bias is not subject to temporal variation.
%\todone{and sentence below. Done, I think}
A similar strategy can also be used to capture variation at the level of individual latent dimensions, e.g.
\begin{equation}
    \gamma_{u,k}(t) = \gamma_{u,k} + \alpha_{u,k} \cdot \text{dev}_u(t) + \gamma_{u,k,t}.
    \label{eq:temporalGamma}
\end{equation}
Note that the deviation terms in \Cref{eq:temporalBeta,eq:temporalGamma}
%\todo{check all crefs with multiple terms}
add only a single term per user (\Cref{eq:temporalBeta}) or a single term per factor (\Cref{eq:temporalGamma}).
The final term $\gamma_{u,k,t}$ in \Cref{eq:temporalGamma} (a temporally-evolving term applied to a specific factor for a specific user) models highly-local preference dynamics and can be used to model (e.g.)~day-specific variability; this term is however highly expensive (in terms of the number of parameters introduced) so may only be available for some users.

When combined with the carefully-engineered deviation term of \Cref{eq:expressiveDev} this allows the model to capture quite complex dynamics while adding only a modest number of parameters; this basic design philosophy (a carefully-designed parametric model that costs only a small number of parameters) will prove a common theme when designing temporal models (\Cref{sec:lessons}).

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/expressiveDev.pdf}
\caption{Expressive deviation term from \protect\cite{koren2009collaborative}. 
\label{fig:expressiveDev}}
\end{SCfigure}

While the expressive deviation term is effective for users who have few interactions, for users who have more it can be useful to fit a more complex model. To do so, a spline function can be used to model gradual shifts in user biases. A spline function smoothly interpolates between a series of control points via the following function:
\begin{equation}
    \beta_u(t) = \beta_u + \frac{\sum_{l = 1}^{k_u} e^{\gamma|t - t_l^u|}b_{t_l}^u}{\sum_{l = 1}^{k_u}e^{-\gamma|t - t_l^u|}}.
\end{equation}
Here $k_u$ is the number of control points for user $u$ (which grows with the number of ratings that user has entered); $t^u$ is a series of uniformly spaced time-points for each user; and $b_{t_l}^u$ is the bias associated with each control point. This type of interpolation is depicted in \Cref{fig:splines}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/splines.pdf}
\caption{Spline interpolation of temporally evolving user bias. \label{fig:splines}}
\end{SCfigure}

The above term is a reasonably flexible way to capture gradual drift in user preferences, though still can't handle sudden changes. \cite{?} addresses this with a `per day' user bias $\beta_{u,t}$, which can be useful for particular days in which users have a lot of activity. Note that while such a bias is unlikely to be helpful when predicting future events, by modeling outliers in this way, the model can essentially learn to `ignore' events that are not useful for prediction. This too is a common design principle when building temporal models: the goal is not so much to `forecast' \emph{future} trends but rather to adjust and account for \emph{past} events appropriately.

The above ideas captures the essential components described in \cite{koren2009collaborative}, though their work does include an exploratory study of several alternative modeling approaches, including incorporating temporal dynamics into neighborhood models like that of \Cref{eq:similarityRecommender?}.

%It should be noted though that the goal of this model was to be effective on Netflix, rather than to capture temporal dynamics in general settings.
%\todone{Pasted this text in here, see if it fits. Cut this but added one sentence below}

%\todone{Still needs personalized latent factors etc. Done, but only one sentence above}


\subsection{What can Netflix Teach us about Temporal Models?}
%\todo{Capitalization, overall organization}
%\todo{Cite "lessons from the Netflix prize" paper}
The model developed in the above case study involved several decisions that are quite specific to Netflix, and indeed the model was designed with the explicit goal of achieving strong performance on a single dataset. As such, many of the specific choices (such as the specific parametric form of the expressive deviation term) may not generalize to other settings.
Nevertheless, there are several important lessons in the above study that apply general when developing temporal models of user behavior:
\begin{itemize}
    \item First, developing bespoke models around specific types of dynamics is somewhat the norm when designing temporal models. As we will see in the following sections, different temporal models abound to capture dynamics in different settings.
    \item One reason for the proliferation of many different hand-crafted temporal models (rather than more `general purpose' models as we developed in \Cref{?,?}) is that temporal models are \emph{expensive}, in terms of the number of parameters required. As such, models are designed to capture the required dynamics while maintaining model parsimony (i.e.,~using as few parameters as possible).
    \item Modeling temporal dynamics is not about \emph{forecasting} so much as it is about determining how to properly adjust, re-weight, or discount past events.
    \item Some specific lessons from Netflix generalize quite well, especially the importance placed on temporally evolving bias terms: the vast majority of temporal variation often owes to shifts in item popularity, user activity (etc.),~and can be captured via evolving $\beta_i$ or $\beta_u$; temporally evolving latent factors ($\gamma_i$ or $\gamma_u$) 
    %\todo{i comes before u or u comes before i?}
    play less of a role, or otherwise are simply too expensive to model.
\end{itemize}

\begin{figure}
\caption{
The models we discussed when exploring temporal dynamics on \emph{Netflix} teach us several general lessons about building temporal models in other settings.
\label{box:netflixLessons}}
%\todo{Not good caption}
\begin{itemize}
    \item Successful solutions to the \emph{Netflix Prize} highlighted the critical importance of temporal dynamics in recommendation settings.
    %\item 
    While the approaches we explored in \Cref{sec:?} are quite Netflix-specific, they highlight a general philosophy followed when building temporal models: temporal models tend to be carefully designed around the dynamics of specific datasets and applications.
    \item A main focus when building temporal models is that of model \emph{parsimony}: to prevent the parameter space from exploding, one must carefully choose models that capture the desired dynamics with as few parameters as possible.
    \item Temporal dynamics may range from `lofty' concepts such as users becoming nostalgic toward older movies, to more mundane sources of variation, such an overall shift in average ratings across the community. Both are important to model as both explain variance in the data: untangling even mundane dynamics is a critical step toward extracting meaningful personalization signals.
    \item The goal of temporal models is typically not to forecast long-term trends. E.g.~our model of opinion dynamics on \emph{Netflix} tells us nothing about what will be popular next year. Rather the goal is typically to 
    %adjust for 
    %compensate
    account for
    discrepencies across different time periods so that interactions across time can be meaningfully compared. The resulting model thus gives a more accurate sense of \emph{current} rating dynamics, if not future trends.
\end{itemize}
\end{figure}
%\todo{repeats the text somewhat. Could expand the text and call this a summary}
%\todo{Make sure to reference figure}


\section{Other Approaches to Temporal Dynamics}
%\todo{Weird section name}

\subsection{Long-Term Dynamics of Opinions}

Early works that deal with temporal dynamics explore the notion of \emph{concept drift} \cite{widmer1996learning,tsymbal2004problem}; early works on this topic are concerned with systems for classification in settings with temporally-evolving data. 
Simple approaches consist of (e.g.)~taking only a \emph{window} of recent examples during training (much like we saw in \Cref{fig:slidingWindows}). More sophisticated approaches allow the context window size to adapt based on how `stable' a particular concept is; or to reuse concepts that recur periodically; or to distinguish drifting concepts from noise. Models based on these ideas, along with theoretical results, are discussed in \cite{widmer1996learning}, among others.


%terms of concept drift 
%[23, 37, 39]. 
%Such models are able to account
%for short-term temporal effects (‘noise’), and long-term changes
%in user behavior (‘drift’), for example due to the presence of new
%products within a community.

Among temporal techniques for recommendation, early approaches incorporate temporal factors into heuristic techniques, such as the model from \Cref{eq:simPredict}. For example in \cite{ding2005time}, the basic idea is to weight related items in \Cref{eq:simPredict} so that recent interactions are weighted more highly:
\begin{equation}
    r(u,i) = \frac{\sum_{j \in I_u} R_{u,j} \cdot \mathrm{Sim}(i,j) \cdot f(t_{u,j})}{\sum_{j \in I_u} \mathrm{Sim}(i,j) \cdot f(t_{u,j})}.
    \label{eq:timeWeight}
    %\label{eq:simPredict2}
\end{equation}
%\todo{consistent subscripts (comma or not)}
Here $t_{u,j}$ is the timestamp associated with rating $R_{u,j}$, and $f(t_{u,j})$ is a monotone function of the timestamp. For example, relevance can decay exponentially for older items:
\begin{equation}
    f(t) = e^{-\lambda \cdot t}
\end{equation}
%\todo{Connecting sentence or descriptio of results}

Other than the methods we discussed in \Cref{sec:caseStudyNetflixTemporal} for temporal recommendation on \emph{Netflix}, other papers studied the notion of gradually evolving concepts within the context of preferences and opinions.
\cite{godes2012sequential}

%Sequential and temporal dynamics of online opinion

\subsection{Short-Term Dynamics}

Temporal recommendation on graphs via long- and short-term
preference fusion

%\todo{finish}

\subsection{User-Level Temporal Evolution}

Most of the sources of temporal dynamics we've explored so far owe to shifting properties of items (or how items are perceived over time, due to e.g.~nostalgia)

\cite{mcauley2013amateurs}
Modeling the evolution of user expertise through online reviews

%\todo{finish}


\section{Personalized Markov Chains}
\label{sec:markov}

The temporal models we saw in \Cref{sec:temporalRec} directly modeled (or extracted features from) the \emph{timestamps} associated with each interaction. We showed how features extracted from timestamps could include factors like seasonality, the day of the week, or nostalgia effects (etc.)~on the \emph{Netflix} dataset, and later we'll revisit such models in the context of the temporal dynamics of fashion (\Cref{sec:fashionTemporal}).

However in many settings, the best predictor of what a user will do next is simply what they did \emph{last}. For example, if you click on a winter coat, then I should recommend you other winter coats, regardless of whether those items are currently in-season.

Even trivial models such as the item-to-item recommenders which we saw in \Cref{sec:neighborhood} implicitly made this assumption. For example, the recommendations made in \Cref{fig:alsoviewed} are made based purely on the context of what is currently being viewed; the user's historical interactions---or their preferences---are not considered.

\paragraph{Markov Chains}
The assumption described above---that the next action is conditionally independent\footnote{Two variables $a$ and $b$ are said to be \emph{conditionally independent} given a third variable $c$ if $p(a,b|c) = p(a|c)p(b|c)$. Essentially this $c$ `explains' any dependence $a$ and $b$. In the case of Markov Chains, the most recent event is sufficient to explain the next action's dependence on the history.
%\todo{Quick writing}
} of the interaction history given the previous action---describes exactly the setting of a \emph{Markov Chain}. 
%\todo{strictly speaking, I suppose a discrete time markov chain}
%\todo{could have a box about conditional independence}
%\todo{Clean up probability notation}
Formally, given a sequence of interactions (among a discrete set of items $i \in \mathcal I$) $I^{(1)} \ldots I^{(t)}$, a Markov Chain
%\todo{Do I use this notation for the item set?}
assumes that the probability of the next interaction given the history can be written purely in terms of the previous interaction:
\begin{equation}
    P(I^{(t+1)} = i | I^{(t)} \ldots I^{(1)}) = P(I^{(t+1)} = i | I^{(t)})
    \label{eq:markov1}
\end{equation}

\emph{Personalized} Markov Chains generalize \Cref{eq:markov1} by allowing the probability of the next item to depend on both the previous item and the identity of the user $u$. That is, for a given user we have:
\begin{equation}
    P(I_u{(t+1)} = i | I_u{(t)} \ldots I_u^{1}) = P(I_u^{(t+1)} = i | I^{(t)}).
\end{equation}

What this means in practice is just that when predicting a user's next action, our prediction should be a function of their previous action as well as their preference dimensions. 
Most of the time we can ignore the formalism of Markov Chains and more simply state that we are trying to fit a function of the form
\begin{equation}
    f(\overbrace{u,i}^{\clap{\text{score associated with the next interaction}}}|\underbrace{I_u^{(t-1)}}_{\clap{\text{given the user's previous interaction}}}).
    \label{eq:fMarkov}
\end{equation}
That is, whereas our previous models took as inputs a user and an item (i.e.,~$f(u,i)$), or a user, item, and timestamp ($f(u,i,t)$), we now wish to model a user, an item, and the user's previous interaction. We might fit this function via a rating estimation framework (as in \Cref{sec:modelbased}), or a personalized ranking framework (as in \Cref{sec:bpr}), etc.

The key challenge in fitting models of the form in \Cref{eq:fMarkov} is that techniques like matrix factorization can no longer be straightforwardly applied. Whereas we previously modeled user/item interactions by factorizing a $U\times I$ matrix we must now factorize a $U \times I \times I$ \emph{tensor}.

We'll explore this idea by investigating specific
%\todo{Wrong word}
implementations
from case studies in the following section.

\section{Case Studies: Markov-Chain Models for Personalized Recommendation}
\label{sec:markovCaseStudies}
%\todo{Style for case study titles}

Below we describe various attempts to extend latent factor recommendation approaches to incorporate signals from the previous item (though some incorporate additional information such as social signals). The main challenges involved include 

summarized in \Cref{tab:markovTable}.

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Markov-Chain models for personalized recommendation.}
\label{tab:markovTable}
%\todo{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{|XXX|}
\hline
\textbf{Method} & \textbf{Reference} & \textbf{Description}\\
FPMC & \cite{rendle2010factorizing} & The next item should be compatible with the user, as well as the previous item.\\
SPMC & \cite{cai2017spmc} & Extends FPMC, incorporating a social term which states that the next item should be similar to our \emph{friends'} previous items.\\
PRME & \cite{feng2015personalized} & Similar to FPMC, but measures compatibility via similarity in a metric space.\\
Translation-based Recommendation & \cite{he2017translation} & Replaces the fixed user embedding $\gamma_u$ with a translation operation in latent item space.\\
\hline
\end{tabularx}
\end{minipage}
\end{table}
%\todo{Style: table rows and items should end in full stop?}

\subsection{Factorized Personalized Markov Chains}
\label{sec:FPMC}


%\todo{Mention that FPMC is handled by Factorization Machines}

%Cover: Factorized Personalized Markov Chains
An early paper to use Markov Chains for personalized recommendation was \emph{Factorizing Personalized Markov Chains for Next-Basket Recommendation} \cite{rendle2010factorizing}. 
The paper was specifically concerned with predicting what 
Factorized Personalized Markov Chains (FPMC for short) predict what
items a user will purchase next, based on the items in their previous basket; customer basket data from \emph{Rossmann} (a German drugstore) was used to train and evaluate the model.

The basic premise of FPMC is that the contents of the previous basket should help to predict the contents of the next one, but also that the basket contents should be personalized to the user. This is achieved by fitting a function of the form
\begin{equation}
    f(u,i|j)
    \label{eq:fuij}
\end{equation}
where $u$ is a user, $i$ is a potential item to be recommended, and $j$ is an item from the user's previous basket.\footnote{Although the original paper uses basket data, baskets are mostly a complication needed to handle their specific dataset. It is more straightforward to present the work by simply considering item sequences (which is how the method is often adopted by other papers). Where we write a term for a previous item, in \cite{rendle2010factorizing} that expression would usually be replaced by a sum over items in the previous basket}. 

The paper discusses the difficulty of modeling sparse interactions of the form in \Cref{eq:fuij}, and explains how this might be addressed with tensor decomposition. The decomposition used is essentially a generalization of the matrix factorization schemes we saw in \Cref{sec:modelbased}, where $f(u,i|j)$ decomposes into a series of pairwise factors:
\begin{equation}
    f(u,i|j) = \gamma^{(ui)}_u \cdot \gamma^{(iu)}_i + \gamma^{(ij)}_i \cdot \gamma^{(ji)}_j + \gamma^{(uj)}_u \cdot \gamma^{(ju)}_j.
\end{equation}
The three terms above denote the user's compatibility with the next item ($\gamma^{(ui)}_u \cdot \gamma^{(iu)}_i$), the next item's compatibility with the previous item ($\gamma^{(ij)}_i \cdot \gamma_j$), and the user's compatibility with the previous item ($\gamma^{(uj)}_u \cdot \gamma^{(ju)}_j$). In practice, the latter expression cancels out when optimizing the model (which is perhaps intuitive as the expression doesn't include the candidate item $i$), so that the factorization can be rewritten
\begin{equation}
    f(u,i|j) = \overbrace{\gamma^{(ui)}_u \cdot \gamma^{(iu)}_i}^{\clap{\text{user's compatibility with next item}}} + \underbrace{\gamma^{(ij)}_i \cdot \gamma^{(ji)}_j}_{\clap{\text{next item's compatibility with previous item}}}.
    \label{eq:FPMC}
\end{equation}
%\todo{Check that superscripts are consistent}
Intuitively, this factorization simply states that the next item should be compatible with both the user and the previous item consumed. Note that (as indicated by superscripts), item parameters are not shared between the terms $\gamma^{(iu)}$, $\gamma^{(ij)}$, $\gamma^{(ji)}$, i.e.,~we use separate sets of factors when modeling how an item interacts with a user vs.~another item.

%\todo{Long section headings overlap with page numbers in header text. Check.}

Ultimately, the model is optimized using a BPR-like framework (as in \Cref{sec:bpr}), i.e., using a contrastive loss of the form
\begin{equation}
    \sigma( f(u,i|j) - f(u,i'|j) ),
    \label{eq:FPMC_BPR}
\end{equation}
where $i'$ is a sampled negative item that the user did not consume. 

Experiments compare two variants of FPMC.
%, which are essentially ablations. 
Regular Matrix Factorization (MF), includes only the first term in \Cref{eq:FPMC}; the other, Factorized Markov Chains, includes only the second. That is, they model $f(u,i) = \gamma_u \cdot \gamma_i$ and $f(i|j) = \gamma_i \cdot \gamma_j$. These experiments thus measure (albeit on a specific dataset) the extent to which future actions can be explained by the previous action, versus overall historital preferences.

Ultimately FPMC outperforms both variants, though interestingly FMC and MF outperform each other under different conditions. Importantly, FMC is particularly effective in \emph{sparse} settings (i.e.,~few interactions per user/item) whereas MF works better on dense data.
%\todo{Do I ever define ablation?}
%show (vs.~MF and FMC)

\subsection{Socially-Aware Sequential Recommendation}

Just as we saw how Bayesian Personalied Ranking can be augmented by sampling signals from social interactions in \Cref{sec:SBPR}, sequential models like FPMC can also be improved by leveraging social information.

\emph{Socially-Aware Personalized Markov Chains} (SPMC) \cite{cai2017spmc} extend FPMC (and Social BPR) by merging both temporal and social signals. The basic idea is to extend \Cref{eq:FPMC} from FPMC based on the reasoning that the user's next item should be similar to those that their friends have recently consumed:
%SPMC: Socially-Aware Personalized Markov Chains for Sparse Sequential recommendation
\begin{equation}
    f(u,i|j) =
    \gamma^{(ui)}_u \cdot \gamma^{(iu)}_i + 
    \gamma^{(ij)}_i \cdot \gamma^{(ji)}_j +
    |\mathcal S|^{-\alpha} \sum_{(v,k) \in \mathcal S} \overbrace{\sigma(\gamma^{(uv)}_u \cdot \gamma^{(uv)}_v)}^{\clap{\text{similarity between $u$ and $v$}}} \underbrace{(\gamma^{(ik)}_i \cdot \gamma^{(ik)}_k)}_{\clap{\text{next item's compatibility with friend's previous item}}}.
    \label{eq:SBPR}
\end{equation}
%\todo{check rendering}
%Here we are summing over $(v,k) \in \mathcal S$, is the set
Here the set $\mathcal S$ 
%\todo{check notation against SBPR}
consists of only the most recent interactions $k$ by each of $u$'s friends $v$. The first time inside the summation $\sigma(\gamma^{(uv)}_u \cdot \gamma^{(uv)}_v)$ measures the similarity between $u$ and $v$, so that we only consider the effect of social influence if $u$ and $v$ are sufficiently similar. 
The term $\mathcal S^{-\alpha}$ normalizes the expression so that social influence does not saturate the other terms for users with a large number of friends. Note that there is no term $\gamma^{(vu)}$ or $\gamma^{(ki)}$ in \Cref{eq:SBPR} (i.e.,~only one set of representations is learned, rather than asymmetric representations as in \Cref{eq:FPMC}); this is done simply to reduce the number of model parameters.
%
\cite{cai2017spmc} show that including both temporal and social terms improves predictive performance over FPMC and Social BPR.
%\todo{boring ending}



\subsection{Locality-Based Sequential Recommendation}
\label{sec:PRME}

In \Cref{sec:aggregation}, we discussed how different aggregation functions (besides the inner product) could be useful in various contexts. This proves to be the case in various sequential recommendation settings, where sequential actions follow some notion of \emph{locality}.

In \cite{feng2015personalized}, sequential recommendation schemes were studied in the setting of Point-of-Interest recommendation. In such a setting, the context of the previous action is particularly informative, since the following action is likely to be (geographically) close.

If the actual semantics of the problem demand some notion of locality, then arguably similarity in the latent space should also be based on locality (rather than, say, an inner product).

The framework of \cite{feng2015personalized}, \emph{Personalized Ranking Metric Embedding} (PRME) models transition probabilities using an expression of the form:
\begin{equation}
    f(i|j) = -d(\gamma_i - \gamma_j)^2 = - \| \gamma_i - \gamma_j \|_2^2.
    \label{eq:PRME_sequential}
\end{equation}
% where $Z(i)$ is a normalization term:
% \begin{equation}
%     Z(i) = \sum_{k = 1}^{|I|}  e^{- \| \gamma_i - \gamma_k \|_2^2}
% \end{equation}
Note two differences between this model (PRME) and FPMC (\Cref{sec:FPMC}):
\begin{itemize}
    \item The main difference is the use a distance (actually a squared distance) function, so that sequential activities exhibit locality in the latent space.
    \item Unlike FPMC, which used separate latent spaces $\gamma^{(ij)}$ and $\gamma^{(ji)}$ for the next and previous item, PRME uses only a single latent space (which saves parameters).
    % \item Finally, the normalization factor allows the next item to be interpreted as a probability, though this is an implementation detail.
\end{itemize}
Like FPMC, PRME also includes an expression encoding the compatibility between the user and the item (again using a distance function), and also trains the model using a BPR-like framework (i.e.,~including a negative item $i'$ as in \Cref{sec:FPMC_BPR}). Other specific details include an explicit feature encoding geographical distance (based on latitude and longitude), and a temporal feature which down-weights the influence of the sequential term if sequential events are temporally far apart.

Though the authors of PRME argue that Euclidean distance is a more natural way of comparing sequential items (and show that PRME outperforms FPMC for POI recommendation), it should be noted that whether one similarity function is `better' than the other largely depends on the semantics of the specific problem and dataset.

Another paper makes use of a similar model for personalized playlist generation \cite{chen2012playlist}. Like PRME, their model (Factorized Markov Embeddings, or FME) notes that sequential songs in playlists tend to be highly localized, such that a metric embedding is possibly well-motivated. The compatibility function given a user $u$, song $i$, and previous song $j$ takes the form
\begin{equation}
    f(i|u,j) = - d( \gamma^{(\mathrm{start})}_i - \gamma^{(\mathrm{end})}_j )^2 + \gamma_u \cdot \gamma'_i.
    \label{eq:FME_sequential}
    %\todo{f(i|u,j) or f(i,u|j)?}
\end{equation}
Note a few differences between FME and PRME:
\begin{itemize}
    \item FME uses a separate embedding for the next song ($\gamma^{(\mathrm{start})}$) and the previous song ($\gamma^{(\mathrm{end})}$). The basic idea being that songs in playlists should not just be highly local, but should gradually `transition' from one song to the next, so that the `start point' of the next song in latent space should be similar to the `end point' of the previous song (\Cref{fig:translation}).
    \item FME uses a combination of both a distance function (for compatibility with the previous item) and an inner product (for compatibility with the user) in \Cref{eq:FME_sequential}. Again, this demonstrates that the correct choice of compatibility function is highly dependent on problem semantics.
\end{itemize}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/translationIdea.pdf}
\caption{Some sequential models use the principle of translation to model sequential transitions between items. \label{fig:translation}}
%\todo_{Add a second figure for transrec}
\end{SCfigure}

\subsection{Translation-Based Recommendation}
\label{sec:transRec}

A third class of models for sequential recommendation are also based on the principle of translation. \cite{he2017translation} built recommender systems by borrowing ideas from knowledge-base completion.

Several techniques for knowledge base completion are based on the principle of learning low-dimensional embeddings that describe relationships among entities \cite{bordes2013translating,wang2014knowledge,10.5555/2886521.2886624}. The basic idea is to represent both entities and relationships as vectors in a low-dimensional space, such that a relation vector encodes how to `translate' between entities. For example, we might seek to learn vectors describing entities such as \emph{`Alan Turing'} and \emph{`England,'} such that given a vector describing the relation \emph{`born in'} we should have
\begin{equation}
    d(\overrightarrow{\text{Alan Turing}} + \overrightarrow{\text{born in}}, \overrightarrow{\text{England}}) \simeq 0,
\end{equation}
where $d$ is a Euclidean distance.

\emph{Translation-based recommendation} \cite{he2017translation} adapts this type of approach to personalized recommendation. Whereas for knowledge graph completion relations tell us how to traverse the space of entities, in a recommendation setting, \emph{items} fulfil the role of entities, and \emph{users} traverse the space of items. Then, given two items $j$ and $j$ that are consumed in sequence, we should have
\begin{equation}
    d(\gamma_i + \gamma_u, \gamma_j) \simeq 0.
\end{equation}

Training such a model is quite similar to how we trained FME and PRME in \Cref{sec:PRME}: that is, we fit a compatibility function between a user, an item, and a previous item (much like \Cref{eq:FME_sequential}):
\begin{equation}
    f(j|u,i) = \beta_j - \| \gamma_i + \gamma_u - \gamma_j \|_2,
\end{equation}
where $\beta_j$ is incorporated so that the method is capable of capturing overall item popularity as well as preference. \cite{he2017translation} further constrains item representations to live on a unit ball (i.e.,~$\|\gamma_i\|_2^2 = 1$), which was found to be effective in the knowledge graph completion settings above.

Conceptually, the above model corresponds to users following a `trajectory' through their interactions over time (\Cref{fig:translation}).
In principle, this ought to mean that related items (e.g.~sequential songs on in a playlist) should be aligned to form a chain of equally-spaced items in the latent space. In practice, such complex dynamics are unlikely to emerge from the model; rather, like other temporal modeling approaches, the model benefits from its parsimony (i.e.,~it has much fewer parameters than other sequential models, due to the use of only a single latent space) but is still able to capture common sequential patterns even in sparse datasets.


\subsection{FPMC in \emph{Tensorflow}}
%\todo{formatting}

Although several of the above models can be implemented via an appropriately designed Factorization Machine (\Cref{ex:?}), it is worth briefly describing how a sequential model might be implemented `from scratch;' this will be useful when implementing variants that do not straightforwardly map to existing architectures (such as the Factorization Machine of \Cref{sec:factorizationMachine}) or libraries.

Here we implement the Factorized Personalized Markov Chain (FPMC) method from \Cref{sec:FPMC}, though the code can straightforwardly be adapted to implement any of the sequential methods discussed in \Cref{sec:sequential?}.

We build our solution on top of our Bayesian Personalized Ranking implementation from \Cref{sec:BPRTensorflow}. First, when parsing the data, we must be careful to process the timestamp:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
for d in parse("goodreads_reviews_comics_graphic.json.gz"):
    u = d['user_id']
    i = d['book_id']
    t = d['date_added'] # Raw timestamp string
    r = d['rating']
    dt = dateutil.parser.parse(t) # Structured timestamp
    t = int(dt.timestamp()) # Integer timestamp
    if not u in userIDs: userIDs[u] = len(userIDs)
    if not i in itemIDs: itemIDs[i] = len(itemIDs)
    interactions.append((t,u,i,r))
    interactionsPerUser[u].append((t,i,r))
\end{lstlisting}
\end{minipage}

Note the use of the \texttt{dateutil} library to process the timestamp. The original timestamp in this dataset consists of raw strings (e.g.~`Wed Apr 03 10:10:41 -0700 2013'); the operation \texttt{dt = dateutil.parser.parse(t)} converts this to a structured format; this can be used to extract features associated with the timestamp, e.g.~\texttt{dt.weekday()} reveals that this date is a Wednesday, which might be useful for extracting features for temporal models such as those in \Cref{sec:temporal?}.\footnote{While fairly obvious in this instance, determining even such simple properties is difficult for certain date formats.}
To build a sequential recommender, we are mainly interested in determining the \emph{sequence order} of the interactions, for which we call \texttt{dt.timestamp()}; for the date above this returns 1365009041, which represents the number of seconds since January 1, 1970 (`unix time'). Such a time representation, while seemingly fairly arbitrary, is useful when our goal is simply to sort datapoints in order of time, as we do when building sequential recommenders.
%\todo{hastily written}

Next we sort each user's history by time, and augment our interaction data such that each interaction $(u,i)$ includes the previous item $j$. We also add a `dummy' item which acts as the previous item for the first observation:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
itemIDs['dummy'] = len(itemIDs)
interactionsWithPrevious = []

for u in interactionsPerUser:
    interactionsPerUser[u].sort()
    lastItem = 'dummy'
    for (t,i,r) in interactionsPerUser[u]:
        interactionsWithPrevious.append((t,u,i,lastItem,r))
        lastItem = i
\end{lstlisting}
\end{minipage}

Given these augmented interactions, we can modify the model from \Cref{sec:?} to include the additional terms from \Cref{equations}. Here we train in a BPR-like setting (i.e.,~including a sampled negative item $k$), though we could similarly adapt the model for rating prediction following code from \Cref{sec:?}. Omitting a few boilerplate elements, the model equation (\Cref{equations}) becomes:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
gamma_ui = tf.nn.embedding_lookup(self.gammaUI, u)
gamma_iu = tf.nn.embedding_lookup(self.gammaIU, i)
gamma_ij = tf.nn.embedding_lookup(self.gammaIJ, i)
gamma_ji = tf.nn.embedding_lookup(self.gammaJI, j)
# (etc.)
x_uij = beta_i + tf.reduce_sum(tf.multiply(gamma_ui, gamma_iu), 1) +\
                 tf.reduce_sum(tf.multiply(gamma_ij, gamma_ji), 1)
x_ukj = beta_k + tf.reduce_sum(tf.multiply(gamma_uk, gamma_ku), 1) +\
                 tf.reduce_sum(tf.multiply(gamma_kj, gamma_jk), 1)
return -tf.reduce_mean(tf.math.log(tf.math.sigmoid(x_uij - x_ukj)))
\end{lstlisting}
\end{minipage}
%\todo{Fix formatting}

The code above could be straightforwardly adapted to implement other sequential models, such as PRME (\Cref{sec:PRME}) or translation-based recommendation (\Cref{sec:transRec}).

\section{Recurrent Networks}
\label{sec:rnn}

%https://colah.github.io/posts/2015-08-Understanding-LSTMs/
%https://arxiv.org/pdf/1409.2329.pdf

A fundamental limitation of the Markov-Chain-based models we saw in \Cref{sec:markov} is that they have a very limited notion of `memory,' due to the assumption that the next event is conditionally independent of all historical events, given the most recent observation. This assumption may be sufficient in certain scenarios, such as recommendation settings that are highly dependent on the context of the previously clicked item (for example). However, as we begin to model text data (\Cref{chap:text}, or sequence data such as heart-rate logs (\Cref{sec:caseStudyHR}), which have longer-term semantics (such as grammatical structures in a sentence, or even an individual's level of `fatigue' in a heart-rate trace).

\emph{Recurrent Neural Networks} (RNNs) seek to achieve this notion of `memory' by maintaining a `hidden state' during each step.\footnote{Note that `simpler' models attempt to accomplish the same goal within the framework of Markov-Chain-based models; see e.g.~Hidden Markov Models. However Recurrent Neural Networks are more typical of current practice and form the basis of models we develop later.
%Note: Hidden Markov Models also do this...
} The hidden state is a vector of latent variables that somehow capture the `context' that the model needs to know to capture the long-term semantics of the problem. Formally, we can visualize the RNN as taking a sequence of inputs ($x_1 \ldots x_N$), and producing a sequence of outputs ($y_1 \ldots y_N$), and maintaining hidden states that update at each step ($h_1 \ldots h_N$). We might visualize this model as follows:
\begin{center}
% \includegraphics[scale=0.25]{fig/rnn1.pdf}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]
%\todo{>=LaTeX above controls the arrow style. Is it good?}
%  \begin{scope}[start chain=going right,nodes=itemc,every
%  join/.style={-latex,very thick},local bounding box=chain]
%  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
%  {$A$};
%  \end{scope}
%  \node[left=1em of chain,scale=2] (eq) {$=$};

 \node(x01) {$x_{t-1}$};
 \node[right=of x01] (x02) {$x_{t}$};
 \node[right=of x02] (x03) {$x_{t+1}$};
 \node[right=of x03] (x04) {$x_{t+2}$};

 \node[item, above=of x01, yshift=-10pt, minimum width=5mm] (x11) {};
 \node[left=of x11, xshift=10pt] (x10) {$h_{t-2}$};
 \node[item, above=of x02, yshift=-10pt, minimum width=5mm] (x12) {};
 \node[item, above=of x03, yshift=-10pt, minimum width=5mm] (x13) {};
 \node[item, above=of x04, yshift=-10pt, minimum width=5mm] (x14) {};
 \node[right=of x14, xshift=-10pt] (x15) {$h_{t+2}$};
 
%  \node[item, above=of x11, minimum width=5mm] (x21) {};
%  \node[left=of x21, xshift=10pt] (x20) {$h^{l}_{t-2}$};
%  \node[item, above=of x12, minimum width=5mm] (x22) {};
%  \node[item, above=of x13, minimum width=5mm] (x23) {};
%  \node[item, above=of x14, minimum width=5mm] (x24) {};
%  \node[right=of x24, xshift=-10pt] (x25) {$h^{l}_{t+2}$};

 \node[above=of x11, yshift=-10pt] (x31) {$y_{t-1}$};
 \node[above=of x12, yshift=-10pt] (x32) {$y_{t}$};
 \node[above=of x13, yshift=-10pt] (x33) {$y_{t+1}$};
 \node[above=of x14, yshift=-10pt] (x34) {$y_{t+2}$};

%  \draw [->] (x11) -- node[right] {$h^{l-1}_{t-1}$} (x21);
%  \draw [->] (x12) -- node[right] {$h^{l-1}_{t}$} (x22);
%  \draw [->] (x13) -- node[right] {$h^{l-1}_{t+1}$} (x23);
%  \draw [->] (x14) -- node[right] {$h^{l-1}_{t+2}$} (x24);
 
 \draw [->] (x11) -- node[above] {$h_{t-1}$} (x12);
 \draw [->] (x12) -- node[above] {$h_{t}$} (x13);
 \draw [->] (x13) -- node[above] {$h_{t+1}$} (x14);
 \draw [->] (x14) -- node[right] {} (x15);
 
%  \draw [->] (x21) -- node[above] {$h^{l}_{t-1}$} (x22);
%  \draw [->] (x22) -- node[above] {$h^{l}_{t}$} (x23);
%  \draw [->] (x23) -- node[above] {$h^{l}_{t+1}$} (x24);
%  \draw [->] (x24) -- node[right] {} (x25);
 
 \draw [->] (x10) -- (x11);
%  \draw [->] (x20) -- (x21);
 
 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);
 
 \draw [->] (x11) -- (x31);
 \draw [->] (x12) -- (x32);
 \draw [->] (x13) -- (x33);
 \draw [->] (x14) -- (x34);
\end{tikzpicture}
\end{center}
The RNN cell, depicted here as a box, is now responsible for determining how the hidden state should change at each step, and what outputs should be generated.

More complex RNN models repeat this idea across multiple \emph{layers}, so that RNN cells can be stacked, e.g.:
%\todo{improve figures, latex renders too small}
\begin{center}
%OLD
% \includegraphics[scale=0.25]{fig/rnn2.pdf}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]
%\todo{>=LaTeX above controls the arrow style. Is it good?}
%  \begin{scope}[start chain=going right,nodes=itemc,every
%  join/.style={-latex,very thick},local bounding box=chain]
%  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
%  {$A$};
%  \end{scope}
%  \node[left=1em of chain,scale=2] (eq) {$=$};



 \node(x01) {$h^{l-2}_{t-1}$};
 \node[right=of x01] (x02) {$h^{l-2}_{t}$};
 \node[right=of x02] (x03) {$h^{l-2}_{t+1}$};
 \node[right=of x03] (x04) {$h^{l-2}_{t+2}$};

 \node[item, above=of x01, yshift=-10pt, minimum width=5mm] (x11) {};
 \node[left=of x11, xshift=10pt] (x10) {$h^{l-1}_{t-2}$};
 \node[item, above=of x02, yshift=-10pt, minimum width=5mm] (x12) {};
 \node[item, above=of x03, yshift=-10pt, minimum width=5mm] (x13) {};
 \node[item, above=of x04, yshift=-10pt, minimum width=5mm] (x14) {};
 \node[right=of x14, xshift=-10pt] (x15) {$h^{l-1}_{t+2}$};
 
 \node[item, above=of x11, minimum width=5mm] (x21) {};
 \node[left=of x21, xshift=10pt] (x20) {$h^{l}_{t-2}$};
 \node[item, above=of x12, minimum width=5mm] (x22) {};
 \node[item, above=of x13, minimum width=5mm] (x23) {};
 \node[item, above=of x14, minimum width=5mm] (x24) {};
 \node[right=of x24, xshift=-10pt] (x25) {$h^{l}_{t+2}$};

 \node[above=of x21, yshift=-10pt] (x31) {$h^{l}_{t-1}$};
 \node[above=of x22, yshift=-10pt] (x32) {$h^{l}_{t}$};
 \node[above=of x23, yshift=-10pt] (x33) {$h^{l}_{t+1}$};
 \node[above=of x24, yshift=-10pt] (x34) {$h^{l}_{t+2}$};

 \draw [->] (x11) -- node[right] {$h^{l-1}_{t-1}$} (x21);
 \draw [->] (x12) -- node[right] {$h^{l-1}_{t}$} (x22);
 \draw [->] (x13) -- node[right] {$h^{l-1}_{t+1}$} (x23);
 \draw [->] (x14) -- node[right] {$h^{l-1}_{t+2}$} (x24);
 
 \draw [->] (x11) -- node[above] {$h^{l-1}_{t-1}$} (x12);
 \draw [->] (x12) -- node[above] {$h^{l-1}_{t}$} (x13);
 \draw [->] (x13) -- node[above] {$h^{l-1}_{t+1}$} (x14);
 \draw [->] (x14) -- node[right] {} (x15);
 
 \draw [->] (x21) -- node[above] {$h^{l}_{t-1}$} (x22);
 \draw [->] (x22) -- node[above] {$h^{l}_{t}$} (x23);
 \draw [->] (x23) -- node[above] {$h^{l}_{t+1}$} (x24);
 \draw [->] (x24) -- node[right] {} (x25);
 
 \draw [->] (x10) -- (x11);
 \draw [->] (x20) -- (x21);
 
 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);
 
 \draw [->] (x21) -- (x31);
 \draw [->] (x22) -- (x32);
 \draw [->] (x23) -- (x33);
 \draw [->] (x24) -- (x34);
\end{tikzpicture}
\end{center}
In this depiction (and in many treatments of the topic), the model is described only in terms of hidden states: the model receives an input, updates its hidden state, and passes this state to the next timestep and the next layer. The first layer may receive inputs (i.e.,~observed values $x$) while the last layer generates outputs (i.e.,$y$).

When designing this cell, one might consider the types of dynamics required to model state transitions between successive steps:
\begin{itemize}
    \item Based on the current hidden state, what output should be generated?
    \item How should the hidden state change as a function of the input that was just seen?
    \item What part of the hidden state should be preserved, and what part can be discarded?
    \item How can hidden state be preserved across long interaction sequences?
\end{itemize}
 
Below we describe one particular implementation of an RNN cell. Although there is nothing particularly sacred about the particular model we present, it is representative of the overall design approach, in terms of capturing the features described above.
%\done{Needs some text here and below. Somewhat useless paragraph}

\subsection{The Long Short Term Memory Model}

\begin{figure}
    \centering
\begin{tikzpicture}[
    % GLOBAL CFG
    font= \scriptsize,
    %font=\sf \scriptsize,
    >=LaTeX,
    % Styles
    cell/.style={% For the main box
        rectangle, 
        rounded corners=5mm, 
        draw,
        thick,
        %very thick,
        },
    operator/.style={%For operators like +  and  x
        circle,
        draw,
        inner sep=-0.5pt,
        minimum height =.2cm,
        },
    function/.style={%For functions
        ellipse,
        draw,
        inner sep=1pt
        },
    ct/.style={% For external inputs and outputs
        circle,
        draw,
        %line width = .75pt,
        %minimum width=0.8cm,
        line width = .6pt,
        minimum width=0.8cm,
        inner sep=1pt,
        },
    gt/.style={% For internal inputs
        rectangle,
        draw,
        minimum width=4mm,
        minimum height=3mm,
        inner sep=1pt
        },
    mylabel/.style={% something new that I have learned
        %font=\scriptsize\sffamily
        },
    ArrowC1/.style={% Arrows with rounded corners
        rounded corners=.25cm,
        %thick,
        },
    ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        %thick,
        },
    ]

%Start drawing the thing...    
    % Draw the cell: 
    \node [cell, minimum height =4cm, minimum width=6cm] at (0,0){} ;

    % Draw inputs named ibox#
    \node [gt] (ibox1) at (-2,-0.75) {$\sigma$};
    \node [gt] (ibox2) at (-1.5,-0.75) {$\sigma$};
    \node [gt, minimum width=1cm] (ibox3) at (-0.5,-0.75) {$\tanh$};
    \node [gt] (ibox4) at (0.5,-0.75) {$\sigma$};

   % Draw opérators   named mux# , add# and func#
    \node [operator] (mux1) at (-2,1.5) {$\times$};
    \node [operator] (add1) at (-0.5,1.5) {+};
    \node [operator] (mux2) at (-0.5,0) {$\times$};
    \node [operator] (mux3) at (1.5,0) {$\times$};
    \node [function] (func1) at (1.5,0.75) {$\tanh$};

    % Draw External inputs? named as basis c,h,x
    \node[ct, label={[mylabel]\emph{cell}}] (c) at (-4,1.5) {\empt{c}{t-1}};
    \node[ct, label={[mylabel]\emph{hidden}}] (h) at (-4,-1.5) {\empt{h}{t-1}};
    %\node[ct, label={[mylabel]left:\emph{input}}] (x) at (-2.5,-3) {\empt{x}{t}};
    \node[ct, label={[mylabel]left:\emph{input}}] (x) at (-2.5,-3) {\empt{h}{t-1}};

    % Draw External outputs? named as basis c2,h2,x2
    \node[ct, label={[mylabel]\emph{cell}}] (c2) at (4,1.5) {\empt{c}{t}};
    \node[ct, label={[mylabel]\emph{hidden}}] (h2) at (4,-1.5) {\empt{h}{t}};
    \node[ct, label={[mylabel]left:\emph{output}}] (x2) at (2.5,3) {\empt{h}{t}};

% Start connecting all.
    %Intersections and displacements are used. 
    % Drawing arrows    
    \draw [ArrowC1] (c) -- (mux1) -- (add1) -- (c2);

    % Inputs
    \draw [ArrowC2] (h) -| (ibox4);
    \draw [ArrowC1] (h -| ibox1)++(-0.5,0) -| (ibox1); 
    \draw [ArrowC1] (h -| ibox2)++(-0.5,0) -| (ibox2);
    \draw [ArrowC1] (h -| ibox3)++(-0.5,0) -| (ibox3);
    \draw [ArrowC1] (x) -- (x |- h)-| (ibox3);

    % Internal
    \draw [->, ArrowC2] (ibox1) -- node[left] {$f^t$} (mux1);
    \draw [->, ArrowC2] (ibox2) |- node[left] {$i^t$} (mux2);
    %\todo{Move less left}
    \draw [->, ArrowC2] (ibox3) -- node[left] {$g^t$} (mux2);
    \draw [->, ArrowC2] (ibox4) |- node[left] {$o^t$} (mux3);
    %\todo{Move less left}
    \draw [->, ArrowC2] (mux2) -- (add1);
    \draw [->, ArrowC1] (add1 -| func1)++(-0.5,0) -| (func1);
    \draw [->, ArrowC2] (func1) --  (mux3);
    %Outputs
    \draw [-, ArrowC2] (mux3) |- (h2);
    \draw (c2 -| x2) ++(0,-0.1) coordinate (i1);
    \draw [-, ArrowC2] (h2 -| x2)++(-0.5,0) -| (i1);
    \draw [-, ArrowC2] (i1)++(0,0.2) -- (x2);

\end{tikzpicture}
    \caption{Caption}
    \label{fig:LSTMcell}
\end{figure}
%\todo{Probably rewrite to avoid h^{<t>} to be consistent with other notation}
% Explain the cell state
%https://colah.github.io/posts/2015-08-Understanding-LSTMs/

The Long Short-Term Memory Model \cite{?} is a specific implementation of an RNN that has been popularized especially for use in text generation tasks (as we'll discuss in \Cref{chap:text}).
%\todo{poorly written}

A challenge in designing RNN cells as described above is how to encourage them to `remember' state across long sequences. To achieve this, the LSTM cell (\Cref{fig:LSTMcell}) preserves the cell state ($c$ in the equation below), mostly unmodified across steps.
Other components are responsible for `forgetting' part of the state ($f$), as a function of the current input and previous hidden state; updating the cell state ($i$ and $g$); updating the hidden state ($h$); and determining what to output ($o$).
Although not particularly relevant for the current discussion---the models discussed here could easily be replaced by different architectures---the specific form of these components in an LSTM cell is as follows:
%\todone{not great}
\begin{eqnarray}
 f^t_l & = & \sigma(W_l^{(f)} \times [h^t_{l-1}; h^{t-1}_l])\\ % forget
 i^t_l & = & \sigma(W_l^{(i)} \times [h^t_{l-1}; h^{t-1}_l])\\ % input gate
 o^t_l & = & \sigma(W_l^{(o)} \times [h^t_{l-1}; h^{t-1}_l])\\ % output
 g^t_l & = & \tanh(W_l^{(g)} \times [h^t_{l-1}; h^{t-1}_l])\\ % input gate
 c^t_l & = & g^t_l \odot i^t_l + c^{t-1}_l \odot f^t_l\\ % cell state
 h^t_l & = & \tanh(c^t_l) \odot o^t_l. % hidden
 %\todone{Called \tilde{c} in the figure, fix}
 %\todo{Nothing in the figure has layers}
\end{eqnarray}

Several variants of the above have been proposed to incorporate additional components, mostly consisting of specific differences to 
%the specific activations above 
how the state is preserved and transformed among the equations above. It should be remarked though that the overall view of a `cell,' in terms of input, outputs, and hidden state, is largely interchangeable across models in spite of differences in specific details.




\section{`User-Free' Sequential Models}

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Summary of user-free recommendation models.}
%\todo{could be two tables?}
\label{tab:userFree1}
%\todo{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{|XXX|}
Method & Reference & Description\\
%\todo{column ordering across tables}
Sparse Linear Methods (SLIM) & \cite{ning2011slim} & Each user is associated with a linear model weighting their interactions over past items; sparsity-inducing regularizers are used to deal with the large number of model parameters.\\
Factored Item Similarity Models (FISM) & \cite{kabbur2013fism} & Replaces the user term in a latent factor model with a second term that represents the user by averaging over item representations from their history. \\ 
%\todo{mouthful}
item2vec & \cite{?} & \\
%\todo{put this in here, need to move method or table. Or move to sequential recommendation table}
%\todo{Or maybe with sequential models? Or maybe just drop this table}
\hline
\end{tabularx}
\end{minipage}
\end{table}
%\todo{Table formatting}
\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Summary of deep-learning based sequential models.}
%\todo{could be two tables?}
\label{tab:userFreeSequence}
%\todo{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{|XXX|}
Method & Reference & Description\\
%\todo{column ordering across tables}
? & \cite{hidasi2015session} & Item sequences are passed to a recurrent network; the network's hidden state is used to predict the next interaction.\\
Neural Attentive Recommendation (NARM) & \cite{li2017neural} & Combines an RNN \cite{hidasi2015session}, but with the addition of an attention mechanism that operates on the sequence of network latent states in order to `focus' on relevant interactions.\\
Self-Attentive Sequential Recommendation & \cite{kang2018self} & Similar to NARM, but uses the principle of \emph{self-attention} (i.e.,~a Transformer model) rather than a recurrent network.\\
BERT4Rec & \cite{sun2019bert4rec} & Also uses the principle of self-attention, though with a different architecture (based on BERT \cite{devlin2018bert}).\\
\hline
\end{tabularx}
\end{minipage}
\end{table}

When introducing sequential models based on Markov Chains in \Cref{sec:markov?}, we argued that it was valuable to model \emph{both} compatibility between the user and the item, as well as the compatibility between adjacent items in a sequence. In short, both sources provide useful and complementary signals that each explain variation in users' activities.

However it is unclear whether this argument applies in the limit. Clearly, a single previous interaction does not provide the full context of a user's preferences; however, a model that includes \emph{enough} historical interactions can possibly capture all of the necessary context without explicitly including any user term.

Below we describe several attempts to model users' historical interaction sequences without explicitly including a user term. Doing so is conceptually appealing as an alternative to the cold-start models we discussed in \Cref{sec:?}: by eliminating the need for an explicit user term, we can quickly make effective recommendations by observing a few user actions, without needing to retrain the model (to fit user terms) or rely on side-information.

As we see below (\Cref{sec:RNNSeqRec}), this type of `user-free' sequential model is a growing trend in deep learning-based recommendation. Borrowing ideas from natural language processing, recent sequential models are able to model dynamics across long user histories without needing an explicit user term.

\subsection{Sparse Linear Methods (SLIM)}
%\todo{Why is SLIM here? It isn't a temporal model. Probably move to previous chapter. Put in same place as FISM}

%While not specifically a sequential model, 
A direct way to avoid including an explicit user term (i.e.,~$gamma_u$) is to describe all of a user's interactions in terms of a binary feature vector enumerating which items they have interacted with (i.e.,~a vector of length $|\mathcal I|$). To predict the score associated with an item $i$, we can then train a linear model (again with $|\mathcal I|$ parameters), much as we did in \Cref{chap:basics}:
\begin{equation}
f(u,i) = R_{u} \cdot W_i
\label{eq:slimEq0}
\end{equation}
Here $R_u$ is a (sparse) vector describing all of a user's interactions, 
i.e.,~equivalent to a row of the interaction matrix $R$ from \Cref{eq:RRR?}. 

Fitting such a model na\"ively is not straightforward, given the high dimensional feature and parameter vectors. 
\cite{ning2011slim} attempts to fit this type of model by exploiting the specific sparsity structure of the vector $R_{u}$, noting that \Cref{eq:slimEq0} can be rewritten in terms of just the items $I_u$ that the user has interacted with:
\begin{equation}
 f(u,i) = \sum_{j \in I_u} R_{u,i} W_{i,j}.
    % f(u,i) & = & R_{u} \cdot W_i\\
    %       & = & \sum_{j \in I_u} R_{u,i} W_{i,j}
           %\todo{Check 2nd bit}
           \label{eq:slimEq}
\end{equation}
%Here $R_u$ is a (sparse) vector describing all of a user's interactions, 
%i.e.,~equivalent to a row of the interaction matrix $R$ from \Cref{eq:RRR?}. 
Here $W$ is an $|I|\times |I|$ parameter matrix which essentially measures item-to-item compatibilities (or similarities).

Conceptually, 
\Cref{eq:slimEq} 
%\todo{Figure out why equation above points to the wrong place}
is similar to the simple heuristic we developed in \Cref{eq:weightedCollabThing}, in which we predicted a rating using a weighted average of previous ratings, where the weighting function was determined by an item-to-item similarity measure (such as the Cosine similarity). Essentially, SLIM follows the same reasoning, but replaces the heuristic item-to-item similarity of \Cref{eq:weightedCollabThing} with a learned matrix $W$.
%\todo{Maybe motivate the model with this argument rather than putting it here}

The main challenge in fitting $W$ is that it has dimension $|\mathcal I| \times |\mathcal I |$.
%\todo{notation}
Were $W$ a dense matrix (i.e.,~every item interacts with every item) training and inference would be expensive; this is circumvented by using a regularization approach which ensures that $W$ is sparse.\footnote{Though in practice experiments are conducted with moderate item vocabularies (e.g.~$|\mathcal I| \simeq 50000$),
%\todo{notation}
and various computational tricks are used to allow for parallelization and efficient inference.}
Sparsity is achieved via a regularization strategy which includes both an $\ell_2$ and $\ell_1$
%\todo{notation}
regularizer:
%, the latter of which induces sparsity as we introduced in \Cref{sec:?}:
\begin{eqnarray}
    \argmin_W & \| R - R W^T \|_2^2 + \lambda \Omega^{(2)}(W) + \lambda' \Omega^{(1)}(W)\\
    \text{s.t.} & W_{i,j} \geq 0;\quad
    W_{i,i} = 0.
    \label{eq:slimEq2}
\end{eqnarray}
%\todo{Do I ever use the \| \| notation?}
%\todo{Don't love the Omega (1) (2) notation}
Note that $\| R - RW \|_2^2$ is merely a matrix shorthand for the predictions made for all interactions $(u,i)$ following \Cref{eq:slimEq}. The first constraint in \Cref{eq:slimEq2} ensures that all terms in the weighting function are positive; the second constraint ($W_{i,i} = 0$) ensures that each item $i$'s rating is predicted only based on interactions with \emph{other} items $j$. $\Omega^{(1)}$ is an $\ell_1$ regularizer (i.e.,~$\Omega^{(1)} = \sum_{i,j} | W_{i,j}$); as we discussed in \Cref{sec:?}, $\ell_1$ regularization leads to sparsity of the matrix $W$.
%\todo{Probably just write out the regularizer to avoid notation}

\cite{ning2011slim} discuss various merits of the above approach. Notable is the rapid inference time (i.e.,~the rate at which recommendations can be made) compared to standard recommendation approaches, and also the \emph{long-tail} performance of the approach. For the latter, compared to (e.g.)~latent-factor approaches---whose representations tend to favor whichever types of items predominate in the data, and fail to capture the dynamics of rarer items---SLIM maintains (relatively) good performance even for tail items.
%\todo{clunky sentence}

\subsection{Factored Item Similarity Models (FISM)}
%\todo{FISM and related models (and exercises) could be moved to "structured" chapter}

Factored
%\todo{I think should be factored, not factorized, check elsewhere}
Item Similarity Models \cite{kabbur2013fism} attempt to directly apply the above
%\todo{Not really "above" anymore}
reasoning, by replacing the user term $\gamma_u$ in a latent factor model (\Cref{eq:latentFactor?}) with a term that aggregates item representations from a user's history. Specifically, the user term in \Cref{eq:latentFactor?} is replaced with an average over \emph{item} terms for all items consumed by that user:
\begin{equation}
    f(u,i) = \alpha + \beta_u + \beta_i + \frac{1}{|I_u|} \sum_{j \in I_u} \gamma'_j \cdot \gamma_i
    %\todo{is it called }
    %\todo{I assume this term should be adjusted to exclude i from j \in I_u?}
    \label{eq:fismEq}
\end{equation}
(recall that $I_u$ is the set of items consumed by user $u$). Note that the item term $\gamma_i$ is separate from the term used to average user actions $\gamma_j$, i.e.,~the model learns \emph{two} sets of latent factors per item.

Spiritually, the average $\frac{1}{|I_u|} \sum_j \gamma'_j$ fulfils the same role as $\gamma_u$, by summarizing the dimensions that are compatible with a particular user.

\cite{kabbur2013fism} considers variants of \Cref{eq:fismEq} for both rating prediction and ranking problems (i.e.,~to optimize the MSE as in \Cref{sec:?} or the AUC as in \Cref{sec:?}).

\cite{kabbur2013fism} argues that the above approach is particularly useful in \emph{sparse} datasets (presumably, datasets where \emph{users} have few associated interactions, while items have several). That is, a traditional latent factor model as in \Cref{eq:latentFactor?} would struggle to meaningfully fit $\gamma_u$ for a user who has only a few interactions; whereas if item histories are denser, a reasonable estimate of user preferences can be estimated by averaging over item terms. Indeed, experiments in \cite{kabbur2013fism} show that the settings in which FISM are effective are closely related to dataset sparsity.


%FISM 

\subsection{Neural Network-Based Approaches}
\label{seq:RNNSeqRec}

A variety of modern approaches to modeling temporal and sequential dynamics of user behavior are built on top of neural network models. Although we won't present a full treatment of such approaches here (since they depend on techniques and models quite different from those covered in this book), here we outline the general directions explored by a few representative examples.

Neural network-based approaches have emerged as the state of the art for a variety of problems in natural language processing (as well as in computer vision, as we discuss in \Cref{chap:visual}, and elsewhere).
%\todo{funny wording}
Techniques range from recurrent neural networks (similar to those we introduced in \Cref{sec:?}), to recent approaches based on self-attention and transformer architectures. 


%Broadly speaking,
Conceptually,
the appeal of neural network approaches to sequential modeling is that they allow models to capture complex \emph{syntax} or \emph{semantics} in sequential data.
%\todo{Organization not so good, since this comes after LSTMs have been described. Probably adjust}
So far, when developing temporal and sequential models we have mostly focused on either of (a) developing simple parametric functions to capture long-term dynamics, as we did in \Cref{sec:caseStudyNetflixTemporal}; or (b) modeling sequential dynamics using the context of previous interactions, following a Markov chain-like setup, as in \Cref{sec:markov?}. Both approaches have limitations: the former tends to require carefully hand-crafting models around specific dynamics, while the latter potentially captures only limited context (e.g.~a single previous interaction). In principle, the techniques below aim to address these issues, by automatically uncovering both long- and short-term dynamics simultaneously, or potentially capturing complex interrelationships between items in a sequence.
%\todo{Check all instances of (a) (1) etc., maybe drop}


Below we describe a few specific instances of models that use neural networks to model user interaction sequences. However ultimately the specific models and architectures are not particularly import: our intention here is to highlight the general approach by which state-of-the-art models, from sequence mining or natural language processing, can be adapted to model user interaction sequences.
%\todo{too similar to opening paragraph}
%\todo{not great paragraph}


\paragraph{Recurrent Networks}
%\todo{paragraph title?}

%\todo{Probably move this to text section, so that it's along side item2vec and similar -- maybe whole section about adapting NLP models}

An early paper to explore the use of recurrent networks for recommendation did so using the types of model we developed in \Cref{sec:?}.\footnote{Specifically they used a Gated Recurrent Unit, or GRU, though spiritually the approach is similar to the LSTM-based techniques we explored in \Cref{sec:?}}.
%
\cite{hidasi2015session} explored the problem of \emph{session-based} recommendation, where user interactions are divided into distinct `sessions' (typically using some heuristic based on interaction timestamps). This setting is typical for recommenders based on recurrent networks (or more broadly, for recommenders that borrow techniques from natural language), since it allows sessions to be treated as analogous to \emph{sentences,} i.e.,~short sequences of discrete tokens (items).

%\todo{Needs to be after item2vec}

Conceptually, the method is fairly similar to \emph{item2vec} (\Cref{sec:?}). Recall
%\todo{only makes sense if after item2vec}
that item2vec learns representations of items such that adjacent items in a sequence (e.g.~items that were both clicked on during a short period) have compatible representations. Likewise, the method from \cite{hidasi2015session} seeks to pass sequences of items into a recurrent network (similar to that of \Cref{sec:?}) such that the hidden state of the network is capable of predicting the next interaction.\footnote{The specific architecture used to achieve this passes the final hidden state into a feed-forward network (similar to those we studied in \Cref{sec:?}), whose final layer estimates scores associated with all items; while computationally expensive, this is feasible for datasets with relatively small item vocabularies, e.g.~in the low tens-of-thousands for data in \cite{hidasi2015session}.}
%\todo{clunky sentence in footnote}

Note that this method (along with most of the related methods below) does not learn a user representation: rather the user `context' is captured via the hidden state of the recurrent network. As such, these methods are a form of `contextual personalization' in the sense that there is no user model as such.
%\todo{argument might go above. In text section, could go before item2vec too}
\cite{hidasi2015session} argue that the benefits of this type of approach are that they will work well in situations where long user histories are unavailable: in such cases, techniques like those we developed for the Netflix prize (\Cref{sec:?}) are unreliable as effective user representations ($\gamma_u$) cannot be learned from only a few interactions; \cite{hidasi2015session} suggests the scenario of recommendation on a niche e-Commerce platform.
%\todo{e-Commerce or e-commerce?}
This possible benefit of `user-free' sequence modeling approaches is put into more context in our broader discussion of the benefits of deep learning-based recommendation in \Cref{sec:?}.
%\todo{maybe rewrite the above depending on where this discussion is}
%\todo{Make sure this point about user-free models is actually in that discussion}

\paragraph{Attention Mechanisms}

The main argument in favor of recurrent network-based approaches is that they can potentially capture longer-term sequential semantics compared to (e.g.)~the Markov chain-based approaches we studied in \Cref{sec:?}. On the other and, the fact that simple Markov chain-based models are so effective in the first place suggests that even the context of a single recent item might be enough to capture a user's `context' in many cases.

The basic 
%goal of 
intuition behind
incorporating \emph{attention mechanisms} into sequential recommender systems is to help the model to `focus' on a small set of interactions among a relatively longer interaction sequence. Intuitively, this ought to allow the model to capture context from long interaction sequences, while still leveraging the fact that the \emph{relevant} part of the context may only consist of a few interactions
\cite{kang2018self}. Attention mechanisms have been used in other settings such as image captioning
\cite{xu2015show}
and machine translation
\cite{bahdanau2014neural}
(among others), where a small component of the input is `attended on' when generating part of the output.



%\todo{Promote deep learning methods to a section, put in text chapter}
\paragraph{Neural Attentive Recommendation}
Neural Attentive Recommendation (NARM) \cite{li2017neural} essentially seeks to extend and combine ideas recurrent recommendation with the use of an attention mechanism. Their suggested model has two main components: a \emph{global} encoder, 
%https://arxiv.org/pdf/1711.04725.pdf
which models sequential data from sessions in much the same way as a traditional RNN-based model (as described above); here the final hidden state of the RNN is responsible for (generating features to) predict the user's next action. A second component, the \emph{local} encoder, uses an attention mechanism over \emph{all} of the hidden states in the sequence to build a feature. %Conceptually, the global encoder's representation describes the semantics of the entire action history, while the local encoder
Roughly speaking, the local encoder's representation captures those specific actions in the user's recent history that best capture the user's intent.
%\todo{Wrote long after previous paragraph, check for consistency}
%\todo{Figure from narm paper is fine}

\paragraph{Self-Attentive Sequential Recommendation}
%SASRec \cite{kang2018self}
%
The approaches above essentially treat attention mechanisms as an \emph{additional} component to extend existing recurrent models (i.e.,~NARM uses an attention mechanism to learn features from the latent representations from an RNN). 
A recent trend in natural language processing has been to rely more heavily on attention modules
to capture complex structures in sentences,
%and to retrieve
%relevant words (in the source language) for generating the next
%word (in the target language)
again with the attention mechanism retrieving relevant words in a source sentence to generate words in a target sentence (e.g.~for machine translation). 
Recently, attention-based sequence-to-sequence methods, especially the \emph{Transfomer} model \cite{vaswani2017attention}, have emerged as the state-of-the-art for various general-purpose language modeling tasks.
%achieved state-of-the-art performance and
%This reliance on attention, self-attention 
The overall goal of such models is much the same as we saw when developing recurrent networks in \Cref{sec:?}: to estimate an output sequence on the basis of an input sequence (or in the case of modeling interaction sequences, estimating the next item in the sequence based on the previous items). Whereas the recurrent networks we developed in \Cref{sec:?} did so by successively passing and modifying a latent state across across network cells, the Transformer
%\todo{style}
architecture eschews this, relying purely on attention modules, whereby the estimation of the next item in a sequence is a function of all previous items,\footnote{Up to some fixed maximum length.} with attention determining which of the previous items is \emph{relevant} to the next prediction.\footnote{This description of the Transformer architecture is admittedly very rough, though a full description is quite involved; see the original paper \cite{vaswani2017attention} or one of many excellent tutorials that attempt to distill the core idea.}

%\todo{Make figure of different attention-based models}

\cite{kang2018self} sought to apply the principle of self-attention to sequential recommendation problems; the method is a fairly straightforward adaptation of the transformer architecture, whereby the next item in a sequence of user interactions is predicted on the basis of the previous items, with an attention module being responsible for determining which of the previous interactions are relevant to predicting the next one. 

Other than the performance benefits of Transformer-based models (compared to recurrent networks and Markov chain-based models), \cite{kang2018self} attempt to visualize the attention weights of their model in order to understand which previous interactions the model `attends on' when 
%making each prediction. 
predicting the next item in a sequence.
Not surprisingly they find that the most relevant item 
%when making the next prediction 
tends to be the previous item in the sequence, though interestingly, significant weight is given to more distant interactions. On some datasets these weights decay rapidly, while on others (such as \emph{MovieLens}), relevance weights are more widely distributed across several historical interactions; collectively this suggests that while recommendations are subject to sequential context, this `context' requires more than a single previous observation to capture.

%\paragraph{Attention Mechanisms}

%
\paragraph{BERT4Rec}
%Following the trend in 
%\cite{kang2018self} of applying state-of-the-art language models to build sequential recommenders, 
%BERT4Rec \cite{sun2019bert4rec} adapts BERT \cite{devlin2018bert}, which (like the model above) 
While \cite{kang2018self} is based on a `standard' Transformer implementation, various extensions have been made to this architecture. For example, BERT4Rec \cite{sun2019bert4rec} adapts BERT \cite{devlin2018bert}, another Transformer-based model architecture. Without going into detail, this model is largely an architectural modification to the approach above; again their superior performance in language modeling tasks appears to translate well to sequential recommendation problems.
Overall this (and other related approaches) further highlight the general trend of applying state-of-the-art language models to build sequential recommenders.
%\todo{not great}

\paragraph{Attentional Factorization Machines}
%\todo{Reference this in a footnote or something from deep learning section in recsys chapter}
We also note that attention mechanisms have been applied in the context of traditional recommender systems.
%, such as factorization machines (which are not necessarily sequential).
%\todo{style}
Attentional Factorization Machines
\cite{xiao2017attentional} apply attention mechanisms to factorization machines, as introduced in \Cref{chap:contentStructure} (recall that such models are not necessarily sequential, though can encode sequential features). The application of attention mechanisms is fairly straightforward. Recall from \Cref{chap:contentStructure} that factorization machines aggregate pairwise interactions across (latent representations of) all features, as in \Cref{eq:?}.  \cite{xiao2017attentional} uses attention to determine which pairwise interactions are most important in a particular context (and arguably helps the model not to focus on irrelevant interactions). Experiments show improvements over traditional factorization machines, and some of the deep learning-based recommendation approaches studied in \Cref{sec:?}.



\paragraph{Summary of Neural Network-Based Sequential Recommenders}
%\todo{not great paragraph title}
Above we have only given a limited sample of recommendation methods based on recurrent networks and attention mechanisms. Although we have provided little detail about the specifics of each approach, we have hopefully highlighted the general trend of borrowing 
%state-of-the-art 
models from natural language and repurposing them for recommendation.
The use of general-purpose language and sequence models for recommendation is increasingly representative of the current state-of-the-art;
we'll further visit this idea as we explore additional language modeling approaches in \Cref{chap:text}.
%\todo{Delete sentence if move to text chapter}
%\todo{repetitive compared to above paragraph}

Note also that the above models in general have no explicit user representation: the input to the model is a sequence of items, based on which the next item is predicted. As such they can be thought of as contextual or memory-based recommendation (though they do have \emph{item} representations which are analogous to $\gamma_i$ in traditional models). Extending the analogy to natural language, a sequence of items is essentially a `sentence' made up of discrete tokens. In principle, so long as long enough sequences are modeled, we can still capture the broad historical interests of the user, without a need for an explicit user term.
%Sequential Recommendation Via Personalized
%Transformer
In many cases the lack of a user term is a virtue of this type of model: when dealing with new users, we can make reasonable predictions based on a few sequential actions, without needing to resort to complex cold-start approaches, or retraining the model (though of course, attempts have been made to to explicitly incorporate user terms into such sequential models, see e.g.~\cite{wu2020sse}).





%\todo{Anything about session-based models? I think one of the user-free models is session based, so maybe discuss the idea there}

%Note that
We will further revisit neural network based approaches and various points throughout the book. In \Cref{chap:text} we consider neural network approaches to modeling text (including for text generation), following the same types of approaches we developed in \Cref{sec:rnn?}. We also consider text representation approaches in \Cref{sec:word2vec}, which can in turn be used to develop \emph{item} representations for use in sequential recommendation (\Cref{sec:item2vec}). In \Cref{chap:visual} we explore convolutional neural network approaches for image representation and generation

\section{Case Study: Personalized Heart-Rate Modeling}
\label{sec:caseStudyHR}

Other than building personalized models of events as in \Cref{sec:?},
%\todo{check}
personalized sequence models can also be used to model various other types of sequence data. In \Cref{chap:textPersonalized} we'll explore using sequence models for personalized text generation; here we briefly examine how such models can be used to model heart-rate sequences.

\cite{ni2019modeling}
explored using personalized sequence models to estimate users' heart-rate profiles as they exercised using data from \emph{EndoMondo}.
%\todo{How do I cite url? Footnote?}
That is, given the GPS route (latitude, longitude, and altitude) that a user intends to run (or walk, cycle, etc.), and a historical record of the user's previous activities, the goal of the model is to forecast the user's heart-rate profile as accurately as possible. 

Modeling the dynamics of heart-rate sequences is particularly challenging for a few reasons, including:
\begin{itemize}
  \item The dynamics are noisy, complex, and are highly dependent on external factors (e.g.~the user's running speed, the altitude, etc.). 
  %A large amount of training data is required to capture these dynamics.
  \item These factors are a combination of both long- and short-term dynamics, e.g.~the user gradually becoming fatigued, versus the user encountering a hill.
  \item There is significant variation among individuals, to the point that a \emph{non-}personalized model would be ineffective for the task.
  \item A large amount of training data is likely required to capture the complex factors involved in heart-rate dynamics. On the other hand, very few observations are likely to be available for each individual.
\end{itemize}

The model used by \cite{ni2019modeling} is fairly similar to the personalized language models we studied in \Cref{sec:?}, where a recurrent neural network (an LSTM in \cite{ni2019modeling}) is augmented with low-dimensional embeddings capturing user characteristics, and contextual features describing the current activity: these embeddings are essentially analogous to user/item embeddings $\gamma_u$ and $\gamma_i$. The argued benefit of low-dimensional user representations in this context is that one can learn a `global' model to capture complex heart-rate dynamics from a large amount of data, along with a small number of user-specific parameters that allow the model to adapt to specific user characteristics from a limited number of observations. Again this is similar to personalized language models, where a high-dimensional language model is adapted via low-dimensional user and item terms.

In addition to user and contextual features, the recurrent network in \cite{ni2019modeling} takes as input the GPS trace (latitude, longitude, altitude) of the route the user intends to complete. This sequence of variables is passed as an additional input to the model during each network step, so that the goal is to forecast heart-rate profiles as a function of the user, contextual variables (weather, time, etc.)~and the intended route.
%\todo{a little repetetive compared to above para}


\cite{ni2019modeling} show the benefit of such a model, over non-personalized alternatives and various alternative model architectures. Ultimately they 
argue that
a system that can accurately make such forecasts can be used in various ways, e.g.
\begin{itemize}
    \item To recommend routes that will help a user to achieve a desired heart-rate profile.
    \item To recommend alternative routes that are semantically similar (in terms of heart-rate dynamics) to those they normally take, e.g.~if they want to maintain their routine while visiting a new city.
    \item To detect anomalies in the event that a user's heart-rate should significantly exceed the projected rate at any point.
\end{itemize}

%\todo{mention why we're talking about this model}

\section{History of Personalized Temporal Models}
%\todo{History sections should have similar title styles}

Although the \emph{Netflix Prize} was one of the early drivers for the use of temporal models in recommendation scenarios, the basic idea of 
%\todo{italicization}
\emph{concept drift}, in which the distribution of labels changes over time, dates back significantly further. For example, the \emph{Dynamic Weighted Majority} algorithm \cite{kolter2007dynamic} considers using an ensemble of classifiers (`experts'); the weighting of these classifiers' decisions changes over time as a function of the empirical performance of the classifiers. \cite{kolter2007dynamic} also gives a historical perspective on the problem of concept drift, dating back to early papers on the topic \cite{schlimmer1986incremental}.

While \cite{koren2009collaborative} showed the effectiveness of temporal factors on the Netflix data,
%\todo{repetitive writing}
a few earlier efforts to use temporal dynamics in recommendation scenarios are noted.
%Adaptive web search based on user profile constructed without any effort from users
For example,
\cite{sugiyama2004adaptive} considered the problem of \emph{personalized search}, though their solution to this problem is actually a form of recommender system. Theirs is an interesting case-study in the use of simple similarity-based methods (like those of \Cref{eq:pearsonSim} and \Cref{eq:simPredict3}) for a problem (personalized search) that doesn't immediately appear to be a recommender systems problem. In terms of temporal dynamics, the main idea is to distinguish between `persistent preferences' and `ephemeral prefernces' when building user profiles (from interaction histories), though this is mostly achieved by considering interactions within differently sized recency windows.

Following the success of modeling temporal dynamics on Netflix, there has been a proliferation of models that attempt to capture temporal dynamics in a variety of settings. As we described in \Cref{sec:?}, temporal dynamics my occur for a variety of reasons, ranging from long- and short-term dynamics \cite{xiang2010temporal}, community effects
community \cite{godes2012sequential}, or 
%\todo{finish}

users \cite{mcauley2013amateurs}

%moved up
%Time weight collaborative filtering
%\cite{ding2005time}




%https://colah.github.io/posts/2015-08-Understanding-LSTMs/



\section{Exercises}

\begin{exercises}
 \item 
 Just as the autoregressive and sliding window techniques we introduced in \Cref{sec:?} can be used for prediction, they can also be used for \emph{anomaly detection}, that is, to determine which events in a sequence substantially deviate from our expectations
 %\todo{style: auto-regres or autoregre, thoughout}
 %Something with auto-regression (if it's in this chapter). E.g.~compare sliding windows vs auto-regression in terms of the MSE.
 \item Our approaches to similarity-based recommendation (write something that uses a similarity-based recommender but considers both user-item and item-to-item similarity)
 
 \item In \Cref{ex:developLatentFactorModelBPR}, we developed a model that included user and item latent factors $\gamma_u$ and $\gamma_i$.
 %\todo{This exercise should be a BPR-like model, not a rating prediction model. Requires that there is a BPR exercise in the previous section. Reference that exercise}
 Adapt the model so that instead of modeling the user, it models the previous item ($\gamma_j$)
 \item In \Cref{chap:sequence} we introduced Factorization Machines as a technique to incorporate various types of features into recommendation approaches (and discussed libraries to do so in \Cref{sec:implicitLibrary}). When introducing Factorized Personalized Markov Chains in \Cref{sec:FPMC} we argued that this type of model can be represented via a factorization machine. Essentially, this can be done by concatenating representations for the item, the user (as in \Cref{sec:facMachine}, and the previous item. Following this approach, using any interaction dataset, compare models that use (a) only the sequential term $\gamma_i \cdot \gamma_j$; (b) only the preference term $\gamma_u \cdot \gamma_i$; and (c) both (i.e.,~FPMC).
 %\todo{Specify whether each problem is rating prediction or implicit feedback}
 \item Likewise, FISM (\Cref{sec:fism}) can be implemented via a factorization machine. Here, the user term is replaced by a feature representing the items the user has consumed (divided by the number of interactions). When extracting a feature based on the interaction history, be sure to exclude the current interaction.
 %\todo{Ugly, replace with an equation to be clearer}
 \item \emph{Herding} effects occur when users' decisions (e.g.~their ratings) are biased by those they've already seen: e.g.~a user may enter a high rating for an item simply because they see that ratings are already high.
 %\todo{finish}
\end{exercises}

\stepcounter{projectCounter}
\subsection{Project \theprojectCounter: Temporal and Sequential Dynamics in Business Reviews}

In this project we'll explore the various notions of temporal dynamics we covered in this chapter, and conduct a comparative study to determine which notions work well in a particular setting. Specifically, we'll consider consumption patterns of business reviews on \emph{Google Local}, which has been used in various studies on temporal dynamics for recommendation (see e.g.~\cite{he2017translation}). Business reviews are useful for the study of temporal dynamics, as activities have a combination of short-term, long-term, and sequential dynamics, though in principle this project could be conducted using any dataset that includes temporal information.

For most of the components in this project, you can base your implementation on the \emph{factorization machine} framework we covered in \Cref{chap:factorizationMachine?}  (e.g.~using the fastFM library as in \Cref{sec:fastFM}).
%\todo{Check formatting and references}
Others which don't follow this framework, such as models based on metric embeddings, are significantly more challenging to implement, and may require solutions based on gradient descent.

We'll explore this problem via the following steps:

\begin{enumerate}
    \item Start by training a (non-temporal) latent factor model for the task. Note that this problem could be cast either as one of rating prediction (as in \Cref{sec:?}) or as visit prediction (as in \Cref{sec:?}).
    %\todo{repeated text from cold-start project}
    Use this initial model to set up your learning pipeline and to find good initial values in terms of the number of latent factors, regularizers, etc. Also consider how you might pre-process the dataset, e.g.~is it better to consier all businesses, or to consider only businesses within a specific category (such as restaurants)?
    %\todo{shoild I include rhetorical questions?}
    If your setting requires negative samples (e.g.~you are modeling visit prediction), consider how you will sample timestamps for the negative items.\footnote{For instance, you could duplicate a timestamp from a positive instance from the same user.}
    \item Next, we'll incorporate temporal dynamics into our model via simple features. 
    Consider how temporal dynamics might be at play in each of these settings, e.g.~ratings might vary on certain days of the week, or be more positive for new businesses, etc.
    Process the timestamps associated with each activity 
    to extract the day of the week, the month of the year, and the `absolute' timestamp (which you may want to scale e.g.~for the range $\lbrack 0, 1 \rbrack$ to represent the lifetime of the dataset).
    Consider any other temporal features that might be useful in this dataset, e.g.~following ideas from \Cref{sec:caseStudyNetflixTemporal}.
    \item Alternately, try including features based on sequential dynamics, e.g.~following the techniques from Exercises \Cref{??}. Again you may use simple features (e.g.~a one-hot encoding of the previous item, as in \Cref{?}), or more complex ones (such as including several recent items, weighting in terms of recency or geographical distance, etc.).
    \item (Hard)
    %\todo{style (HARD)}
    Try to implement an alternate sequential model, such as one based on recurrent neural networks from \Cref{sec:?}. Although challenging to implement `from scratch,' several of the cited papers have public code repositories on which you may base your implementation.
\end{enumerate}

Although the main goal of the project is to build familiarity with and compare various temporal modeling techniques, consider also whether the temporal models you develop give you \emph{insight} into the underlying dynamics of the data. For example, under what circumstances are users guided by temporal dynamics? Or what types of businesses are most subject to seasonal trends? 

\part*{Emerging Directions in Personalized Machine Learning}
%Not a great name for part...

\chapter{Personalized Models of Text}
\label{chap:text}

Throughout this book we have already worked with a wide variety of datasets involving text. Although we have made little use of this data so far, text can be useful both as a feature to improve the predictive performance of the types of method we've seen so far, but can also be used to explore a variety of new applications.

As a feature to improve prediction, effectively making use of text is not straightforward. Text is noisy, varies in length, has complex syntax, and only a small fraction of words may be important to prediction. As such, we start this chapter with a primer on the types of feature engineering techniques that can be used to extract meaningful information from text (\Cref{sec:textBasics}).

Following this, we explore how text can be used to improve the types of personalized models we've developed in previous chapters. In the case of recommender systems, text ought to be helpful, as there is abundant text (e.g.~product reviews) that can help us to `explain' the underlying dimensions of users' preferences and items' properties; however effectively extracting these signals is not straightforward (\Cref{sec:?}).

As a form of sequential data, models from natural language processing can also guide us to better ways to model sequences, following the material we developed in \Cref{chap:temporalSequential}. After introducing a selection of models of natural language, we explore techniques that borrow the same ideas for sequential recommendation (\Cref{sec:item2Vec}).
%\todo{repetitive}

Having explored methods that use text as a feature to improve prediction, we also explore recent trends in natural language \emph{generation}. 
%Methods that generate text 
The proliferation of language models based on recurrent networks, along with a slew of recent architectures for general-purpose language modeling and generation, have opened up a range of applications ranging from goal-oriented dialog \cite{bordes2016learning}
to story \cite{roemmele2016writing} or poetry \cite{zhang2014chinese} generation.
Naturally, such methods benefit from personalization, to better capture the context, preferences, or characteristics of individual users 
\cite{joshi2017personalization,majumder2020like}. We'll explore such settings in the context of recommendation approaches that generate text to `explain' recommendations to users (\Cref{sec:?}), and systems that generate recommendation via natural language conversations with users (\Cref{sec:?}).

Outside of recommendation, we also examine the use of text in other personalized or contextual settings, from simple forms of personalized retrieval (\Cref{sec:?}), to complex systems such as Google's \emph{Smart Reply} (\Cref{sec:?}).
%\todo{style}

%Make the points that (1) many personalization tasks involve text somehow; (2) models from natural language are often used when modeling sequence data -- item interactions etc. can benefit from similar sequence models.

\section{Basics of Text Modeling: The Bag-of-Words Model}
\label{sec:textBasics}

Many of the datasets we've already been working with contain natural language data. Several tasks can potentially benefit from from this data when making predictions, and as we'll see later, text can drive a variety of unique applications.
%\todo{maybe cover this in intro to chapter, otherwise this paragraph is too isolated anyway}

\subsection{Sentiment Analysis}
\label{sec:sentiment}

To understand why modeling textual data is difficult, let's consider the seemingly simple task of predicting a rating based on the text of a review:
\begin{equation}
    \text{rating} = X^{(\text{review})} \cdot \theta
    \label{eq:sentiment0}
\end{equation}
%\todo{Capital X isn't really consistent notation}
This is the same as the type of problem we set up in \Cref{chap:basics}, except that our features $X$ are derived from the reviews. Intuitively it makes sense that reviews should help us predict ratings, as they are specifically intended to explain a user's rating.

The task described in \Cref{eq:sentiment0} captures the basic setting of \emph{sentiment analysis}, i.e.,~learning what types of features are associated with `positive' (i.e.,~high ratings) and `negative' language. The main challenge is how to appropriately extract meaningful features from text.

\begin{figure}
\caption{What's the point of sentiment analysis?}
%TODO: Need citations for this box!
Sentiment analysis, viewed superficially, may seem like an odd (or a `toy') task: why would we ever want to predict ratings from reviews, given that in practice we never observe a review \emph{without} a rating? Nevertheless, sentiment analysis is one of the core topics in natural language processing whose importance transcends the immediate task of predicting ratings. Sentiment analysis research is generally focused on:
\begin{itemize}
    \item Understanding the sociolinguistic
    %TODO: Right word?
    dimensions of sentiment, rather than the immediate task of predicting the rating.
    \item Building sentiment models that are \emph{general purpose}, i.e., models that can be trained on one corpus (e.g.~of reviews), but can be used to predict sentiment in settings where ratings aren't available.
    \item As a benchmarking task to test scalability, and the ability of NLP systems to understand detailed nuances in language.
\end{itemize}
\end{figure}

The first problem that we must deal with is that the features in \Cref{eq:sentiment0} are \emph{fixed length} (i.e.,~$X$ is a matrix), whereas text is sequence data. Later, we'll see how to model sequences more directly in \Cref{sec:RNN},
%\todo{will we?}
but for now let's see how much progress we can make just by extracting a pre-defined set of features from each review.

\emph{Bag-of-words} models attempt to solve this by composing $X$ out of features that encode the presence or absence of certain words in a model. Thus it ignores key details such as the order in which words appear (see \Cref{fig:bowExample}).

\begin{figure}
\caption{Bag-of-Words models}
\begin{center}
\parbox{0.48\textwidth}{ \small Loved every minute. So sad there isn't another! I thought JK really made Harry an even stronger archetypal hero - almost in a Paul Maud'Dib from Dune kind of way. He's fighting the ultimate evil, he's brave and takes risks, and believes in himself and doesn't give up despite many hardships.}\hspace{0.025\textwidth}\parbox{0.48\textwidth}{ \small risks, Paul and hardships. believes the almost sad ultimate kind up every - an there in brave hero I fighting Dune another! way. himself made really he's despite He's Loved from archetypal minute. and a Maud'Dib isn't even evil, of in many stronger So takes JK thought Harry give and doesn't}
\end{center}
\caption{The two reviews above have identical \emph{bag-of-words} representations (the second randomly shuffles the words of the first). The review at right misses details that depend on syntax. Is there still enough information in the review at right to tell whether the overall sentiment is positive? \label{fig:wobExample}}
\end{figure}

A key component in the bag-of-words model is the \emph{dictionary} that is used to build our feature vector. Our first attempt to build this dictionary might merely consist of compiling \emph{every} word in a given corpus, e.g.:
%\todo{Find better way to include code}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
wordCount = defaultdict(int)
for d in data:
  for w in d['review/text'].split():
    wordCount[w] += 1

print(len(wordCount))
\end{lstlisting}
\end{minipage}
%\todo{Do I want print statements in these?}

Doing so on just the first 5,000 of our beer reviews
%\todo{Use this or something else}
reveals a total of \emph{36,225} unique words. In other words, each review (on average) contains around seven previously unseen words. This number sounds large, but we might think that by the time we have seen 5,000 reviews that we have `saturated' the vocabulary of English words, and will not see too many more. However, if we repeat the same experiment for (e.g.)~10,000 reviews, we obtain 55,699 unique words---not quite double, but still a substantial increase, revealing that we do not quickly saturate the vocabulary.

Looking at a few of the actual words in our dictionary, we see words like:
\begin{equation*}
    \text{...the}; \quad \text{09:26-T04.}; \quad \text{Hopsicle}; \quad \text{beery}; \quad \text{\#42}; \quad \text{\$10.65}; \quad \text{(maybe}; \quad \text{(etc.)}
\end{equation*}
That is we see words including proper nouns, unusual spelling, prices, punctuation, etc. From this, we quickly see that we will not quickly run out of unique `words,' given any reasonable number of reviews.

To use a bag-of-words representation, we will need to reduce this dictionary to a manageable size. Some potential steps to do so include:

\paragraph{Removing capitalization and punctuation}
Removing punctuation will reduce our dictionary size substantially, as words like `(maybe' (i.e.,~a word following a parenthesis) will be resolved to a common word. Likewise, we might ignore different capitilization patterns simply by converting all documents to lower-case.

\paragraph{Stemming}
Stemming, i.e.,~resolving similar words to their common word stems. Words like `drink,' `drinking,' and `drinks' would all map to `drink'.\footnote{Or words like `argue,' `arguing,' and `argus' would map to `argu'---the stem need not be an actual word.} Such techniques are mostly motivated by search and retrieval settings (i.e.,~to make sure results are retrieved even if a query uses a different word inflection than the result), though they could also be used to reduce our dictionary size. See e.g.~\cite{lovins1968development,porter1980algorithm} for examples of stemming algorithms.

\paragraph{Stopwords}
Stopwords are common English words that likely carry (relatively) little predictive power compared to their frequency in a document. Standard stopword lists\footnote{See e.g.~\texttt{nltk.corpus.stopwords} in Python.} include words such as `am,' 'is,' `the,' `them,' etc. Removing them can reduce our dictionary size a little, or otherwise prevent our feature representations from being overwhelmed by common words (though we will see other ways to address this in \Cref{sec:tfidf}).

Decisions like whether to remove punctuation, whether to stem, or whether to remove stopwords, are largely dataset and application dependent. Characters like exclamation points could be predictive of sentiment;\footnote{Often, important punctuation characters are preserved by treating them like separate words, e.g.~the string `great!'~would be replaced by `great !'.} or different word inflections (such as `drinks' or `drinker' in a corpus of beer reviews)
%\todo{not the best example}
may have different meanings; or stopwords like `i' and `her' could change the meaning of a sentence.

As such, the procedures above essentially amount to feature engineering choices: we should ultimately accept or reject the above procedures based on whether they improve performance for our given application (again, these are the kind of choices we'd make with our validation set as in \Cref{sec:validation}).

For the moment, let's consider removing punctuation and capitalization, e.g.:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
for d in data:
  r = ''.join([c for c in d['review/text'].lower() if not c in string.punctuation])
  for w in r.split():
    wordCount[w] += 1
\end{lstlisting}
\end{minipage}
%\todo{Indentation (probably four spaces always)}
After removing them, we are left with 19,426 unique words. This is a reduction of nearly half compared to what we had before removing them, but is still a fairly large dictionary size.

A more straightforward way to reduce our dictionary to a manageable size is simply to include only the most commonly occurring words:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
counts = [(wordCount[w], w) for w in wordCount]
counts.sort()
counts.reverse()

words = [x[1] for x in counts[:1000]]
\end{lstlisting}
\end{minipage}
Although perhaps not a completely satisfactory solution, this representation at least allows us to build a feature representation. 
%Put later
%In this case the 1,001$^\text{st}$ most popular word in this example appears in around one percent of all reviews. 
A simple bag-of-words-based sentiment analysis model would now consist of predicting:
\begin{equation}
    \mathrm{rating} = \theta_0 + \sum_{w \in \mathcal D} \mathrm{count}(w) \cdot \theta_w.
    \label{eq:sentiment1}
\end{equation}
Here $\mathcal D$ is our \emph{dictionary} (i.e.,~our set of words); $\theta$ is indexed by a word $w$, but in practice we would replace each word by an index (here from 1 to 1000) for the sake of building a feature matrix.

Fitting such a model on our beer review dataset\footnote{Here with an $\ell_2$ regularizer with coefficient $\lambda=1$, 4,000 data points for training and 1,000 for testing.} yields a training MSE of 0.27 and a testing MSE of 0.51; this attests to both the high accuracy of such models, as well as their ability to overfit.

Examining the coefficients $\theta$, we find the five words associated with the most negative coeficients are:
\begin{eqnarray*}
 \theta_\mathrm{skunk} & = & -0.364;\\
 \theta_\mathrm{oh} & = & -0.312;\\
 \theta_\mathrm{skunky} & = & -0.292;\\
 \theta_\mathrm{bland} & = & -0.284;\\
 \theta_\mathrm{recommend} & = & -0.267,
%(-0.3635808995815085, 'skunk'),
%(-0.31193813774769147, 'oh'),
%(-0.29234181779758006, 'skunky'),
%(-0.2839149980664344, 'bland'),
%(-0.26701385744199396, 'recommend')
\end{eqnarray*}
and the five most positive words are:
\begin{eqnarray*}
\theta_\mathrm{raisins} & = & 0.204;\\
\theta_\mathrm{impressed} & = & 0.224;\\
\theta_\mathrm{keeps} & = & 0.234;\\
\theta_\mathrm{always} & = & 0.256;\\
\theta_\mathrm{exceptional} & = & 0.320
%(0.2043839147952575, 'raisins'),
%(0.22364690773701465, 'impressed'),
%(0.23419283704758836, 'keeps'),
%(0.2564967471468373, 'always'),
%(0.3197211175655953, 'exceptional')
\end{eqnarray*}
For example, every instance of the word `skunk' (a negative association with beer) decreases our prediction of the rating by around one third of a star; likewise every instance of the word `exceptional' increases our prediction by around the same amount. A few more observations:
\begin{itemize}
    \item Words like `skunk' and `skunky' presumably convey the same information, indicating some redundancy in our representation; possibly this could be addressed by stemming.
    \item The word `oh' is extremely negative, despite conveying little meaning by itself. Presumably, it occurs in phrases like `oh no' (whereas `no' itself appears in a wide variety of phrases so is not as negative); possibly we could account for such confounds by using n-grams (\Cref{sec:ngrams}).
    \item Likewise words like `always' or `keeps' are highly positive despite conveying little sentiment in isolation; the word `raisins' possibly appears in the context of a specific popular item.
    \item The word `recommend' is highly \emph{negative}, in spite of seeming to convey positive sentiment. Presumably, this word frequently appears in negative phrases (`would not recommend,' etc.); we could account for such confounds by better handling of negation.
\end{itemize}
Finally, it is notable that the words `skunk' and `exceptional' are the 962$^\text{nd}$ and 991$^\text{st}$ most popular words in our corpus---that is, they are words we nearly discarded when we selected a 1,000 word dictionary. On the one hand, this might be an argument that our dictionary size was too small, since we nearly missed the most important words. On the other hand, it is almost true by definition that the most predictive words will be rare ones: it is, after all, quite exceptional for an item to be described as `exceptional'!

\subsection{N-grams}
\label{sec:ngrams}

Some of the oddities with our sentiment model from the previous section possibly arise due to the simplifying assumptions made by the bag-of-words model. Critically, a bag-of-words model has no notion of \emph{syntax} in a document, and as such cannot handle even simple concepts such as negation (e.g.~`not bad') or compound expressions that carry different meaning together than alone (e.g.~`oh no').

\emph{N-gram} models attempt to address some of these issues by considering words that frequently co-occur in sequence. That is, \emph{bigrams} consist of pairs of words that are adjacent in a document; \emph{trigrams} consist of groups of three words that appear consecutively in a document (etc.). \text{not great writing}

For example, the N-grams associated with the following sentence would be:
\begin{eqnarray*}
\begin{tabular}{ll}
Sentence: & `Dark red color, light beige foam'\\
Unigrams: & [`Dark', `red', `color,', `light', `beige', `foam']\\
Bigrams: & [`Dark red', `red color,', `color, light', `light beige', `beige foam']\\
Trigrams: & [`Dark red color,', `red color, light', `color, light beige', $\ldots$]\\
(etc.)
\end{tabular}
\end{eqnarray*}
From the above example we can already see some potential benefits of using an N-gram representation, e.g.~several of the words in the above sentence are adjectives that modify the words `color' and `foam;' without an N-gram representation we might fail to correctly understand those nouns in context.\footnote{Though one could argue the opposite: if an adjective like `beige' rarely occurs in any other context, then a regular bag-of-words representation may be sufficient to capture the relevant information.} Similarly, negated terms (e.g.~`not bad' etc.)~are readily handled by an N-gram representation.

N-grams are straightforward to extract in Python, e.g.:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
sentence = "Dark red color, light beige foam"
unigrams = sentence.split()
bigrams = list(zip(unigrams[:-1], unigrams[1:]))
trigrams = list(zip(unigrams[:-2], unigrams[1:-1], unigrams[2:]))
\end{lstlisting}
\end{minipage}

Having extracted N-grams, our approach to modeling the data is much the same as that of our bag-of-words model, i.e.,~we extract counts associated with each N-gram, and use those counts as features to predict some outcome.

Note that generally we would not use (e.g.)~bigrams exclusively, but rather would use a representation that included both unigrams and bigrams simultanously. Much like our previous approach from \Cref{sec:sentiment}, which simply found the most popular words (i.e.,~unigrams), we might simply count the popularity of unigrams and bigrams simultaneously.

On the same review corpus as in \Cref{sec:sentiment}, the majority of popular terms are still unigrams---N-grams comprise 452 of the 1,000 most popular terms.

A few of the most popular N-grams include terms like `with a,' `in the,' `of the,' etc. At first glance, these don't look particularly useful, and are mostly combinations of stopwords. Likewise, few of the top N-grams appear to include negation---for example among the top 1,000 terms, only eleven include the word `not', including e.g.~`but not', `not a', `is not', `not much', `not too', etc. Again these seem unlikely to be informative given that they do not modify meaningful terms.

The above example highlights that N-grams are not quite a simple panacea for the issues mentioned above about handling adjectives and negation (etc.). Indeed, one must carefully trade off the fact that N-grams introduce substantial redundancy into our feature vector with the possibility that they introduce some useful compound terms. Note that it is not straightforward to get the `best of both worlds' in this scenario: assuming a dictionary of fixed size, we are potentially discarding informative unigrams (like `exceptional' from \Cref{sec:sentiment}) in favor of less-informative N-grams. Again, these issues are model- and dataset-specific, and likely could be addressed by better accounting for stopwords (or possibly by using a larger dictionary). Mostly this example simply highlights that we must make extra considerations when incorporating N-grams into a model, and that they will not confer benefits unless carefully handled.

Some of these potential advantages and disadvantages of N-gram representations are summarized in \Cref{box:Ngrams}. Many of the disadvantages center around the redundancy of the representation, which can partly be mitigated by carefully selecting \emph{important} features. We will revisit the topic of word important in \Cref{sec:tfidf}

\begin{figure}
\caption{Arguments For and Against N-grams. \label{box:Ngrams}}
To summarize our discussion in \Cref{sec:ngrams}, N-grams are not always beneficial in language modeling tasks, partly because one has to trade of the predictive value of N-grams against the redundancy in encoding them. Some positive and negative points are summarized below:
\begin{itemize}
    \item N-grams can straightforwardly allow us to handle negation, and various forms of compound expressions, 
    \item Our dictionary size quickly multiplies when using N-grams. Assuming that we can handle a fixed dictionary size, in practice this sometimes means some informative unigrams will be replaced by uninformative N-grams.
    \item Stopwords, which made up a small fraction of our unigram dictionary, quickly multiply when building N-gram representations; thus there is additional redundancy in the N-gram representation.
    \item An N-gram representation may add substantial redundancy (or co-linearity) between features, e.g.~as informative unigrams are now duplicated among several N-gram features.
    %TODO: This
    %\item Colinearity / redundancy (maybe two points
    %\item Not useful words?
\end{itemize}
\end{figure}

Finally, let's evaluate an N-gram representation on the same task from \Cref{sec:sentiment}. Again we'll build a `bag-of-ngrams' model by taking the 1,000 most popular N-grams (for any value of N). Once extracted our model is again the same as that from \Cref{eq:sentiment1}, the only difference being that our dictionary consists of a combination of N-grams of different lengths.

After fitting the model, performance in fact slightly degrades compared to the model from \Cref{sec:sentiment} (testing MSE of 0.54 compared to 0.51 in \Cref{sec:sentiment}). Examining some of the most predictive N-grams (i.e.,~largest and smallest values of $\theta_w$) we find terms such as
\begin{equation*}
    \theta_{\text{pitch black}} = -0.397; \quad \theta_{\text{ptich}} = 0.354.
\end{equation*}
Upon further inspection, the word `pitch' \emph{always} appears in the expression `pitch black', to the point that the two terms will mostly `cancel out'; again this highlights an issue of redundancy (and co-linearity, as in \Cref{sec:colinear}) of our representation.
%\todo{rushed prose}

Possibly we can address this simply by further regularizing our model. Increasing the regularization coefficient to $\lambda = 10$ improves the performance somewhat (test MSE of 0.506), and results in more reasonable coefficients, such as:
\begin{equation}
    %\theta_{\text{water}} = -0.232; \quad
    \theta_{\text{wonderful}} = 0.177; \quad
    \theta_{\text{not bad}} = 0.174; \quad
    \theta_{\text{low carbonation}} = 0.137,
\end{equation}
which appear to correctly handle negation (`not bad') and compound words (`low carbonation').

%\subsection{Personalized Sentiment Analysis}

\subsection{Word Relevance and Document Similarity}
\label{sec:tfidf}
%\todo{Is this the right section label?}
%\todo{Worth having this section at all?}

%A question that arose in the previous section was how to prevent our feature representations from being weighed down by uninformative or `irrelevant' words. To investigate this idea further, we need to specify what it means for a word to be `relevant' in a document.
%\todo{mediocre motivation, since we don't really use tf-idf to build dictionaries}

%When building a dictionary in \Cref{sec:sentiment} we simply counted the number of times each word appeared across all documents, i.e.,~we took the most popular words. However, this left us with many stopwords (or otherwise irrelevant words), which (probably not surpringly) are the most common words in most documents.

Suppose we wanted to build a system that recommends articles that are similar in content to ones a user had recently interacted with. As in \Cref{sec:neighborhood}, we might do so by defining an appropriate similarity metric between articles, and recommend those articles that are most similar:
\begin{equation}
    f(i) = \argmax_j \mathrm{Sim}(i,j).
\end{equation}
Given that our goal is to define similarity based on article \emph{content} (rather than interaction histories as in \Cref{sec:neighborhood}), we might consider defining similarity based on the feature representations we developed in \Cref{sec:bagOfWords}. For example, we could compare the cosine similarity of two bag-of-words representations:
\begin{equation}
    \mathrm{Sim}(i,j) = \text{Cosine Similarity}(x_i, x_j).
\end{equation}
%\todo{sim / Sim is not consistently capitalized (in various sections)}
%\todo{Jaccard is italicized (in other section), but should it be?}
%\todo{Never refered to BoW vectors as x_i. Maybe call x^Bow_i or something}
However as we discussed in \Cref{sec:bagOfWords}, the vectors $x_i$ and $x_j$ will be dominated by the most common words in the corpus (i.e.,~the largest magnitude words will likely be stopwords).

In practice a user would probably not regard two documents as `similar' just because they used words like `the,' `of,' or `and' in similar proportions. As such, we presumably want a feature representation $x_i$ that focuses on \emph{relevant} terms.

In \Cref{fig:tfidfexamp} we see an example of a book review (of \emph{Watership Down}) with the most frequently occurring words in boldface. Naturally, we would not say that the topic of this review was `i' or `a,' even though those words are the most frequently occurring.

\begin{figure}
\parbox{0.48\textwidth}{ \small \textbf{I} read \textbf{this} after hearing from \textbf{a} few people that \textbf{it} \textbf{was} among their all-time favorites. \textbf{I} \textbf{was} almost put off when  \textbf{I} \ul{saw} \textbf{it} \textbf{was} \textbf{a} \textbf{story} \ul{\textbf{about}} rabbits, originally written as \textbf{a} tale by \textbf{a} \ul{father} \textbf{to} his \ul{children}---but I'm \ul{glad} \textbf{I} wasn't. \textbf{I} found \textbf{the} folk \ul{tales} \ul{\textbf{about}} El-ahrairah \textbf{to} be very impressive. \textbf{The} author clearly had \textbf{a} vivid imagination \textbf{to} create so much of \textbf{the} rabbits culture \textbf{and} \ul{history}. But \textbf{I} think \textbf{this} book \textbf{was} \ul{worth} reading as it's really \textbf{a} \textbf{story} \ul{\textbf{about}} survival, leadership, \textbf{and} \ul{human} \ul{nature}. Oh \textbf{and} Fiver rocks. \textbf{And} BigWig is \textbf{the} man.}\hspace{0.04\textwidth}\parbox{0.48\textwidth}{I was delighted by this book... the only fault is that it was too short! What a fantastic idea; a refuge for the children who have had adventures \& now cannot fit back into the identity assigned to them. How many of us are not comfortable in the families we were born to? I loved the way the different doorways were sorted; one would think that adventures shared would be a bonding moment. Rivalries will be ever present; guess that is human nature. I don't want to describe too much \& ruin the magic[...]
%this author is the heir apparent to Neil Gaiman. Im so glad she has more books to discover !
}
\caption{Term frequency and \emph{tf-idf} comparison. At left, the top 10 words by term frequency are bolded (i.e.,~the most common words in the review), and top 10 tf-idf words (based on a sample of 50,000 reviews) are underlined. A highly-similar review (based on cosine similarity of \emph{tf-idf} vectors is shown at right. \label{fig:tfidfexamp}}
\end{figure}

Rather, we might argue that the words like `nature' or `children' are more characteristic of the document, presumably because they do \emph{not} occur in most documents. As such, we might consider words characteristic of a particular document to be those that occur frequently in that document but not in others.

To capture such a definition we should separately consider word frequency in a particular document, and frequency across a corpus at large. To do so we define two terms. First, the \emph{term frequency} of a word $w$ in a document $d$ is simply the number of times that word appears in the document:
\begin{equation}
    \text{Term Frequency}(w,d) = \mathit{tf}(w,d) = |\langle t \in d | t = w \rangle|.
\end{equation}
Note that this is essentially the same as the Bag-of-Words representation we developed in \Cref{sec:bagOfWords} (although the latter includes a fixed dictionary size).

Next, we define the \emph{document frequency} of a word across a set of documents in terms of a word $w$ and a corpus $\mathcal D$:
\begin{equation}
    \text{Document Frequency}(w,\mathcal D) = \mathit{df}(w,\mathcal D) = |\langle d \in D | w \in D|.
\end{equation}

Now, for a word to be `relevant' in a particular document, we want the term frequency $\mathit{tf}(w,d)$ to be high, and the frequency of the word across the entire corpus $\mathit{dF}(w,\mathcal D)$ to be relatively low. The \emph{tf-idf} measure (term frequency-inverse document frequency) is a heuristic which achieves this goal via the following function:
\begin{equation}
\mathchardef\mhyphen="2D
    \mathit{tf\mhyphen idf}(w,d,\mathcal D) = \mathit{tf}(w,d) \times \log_2 \left( \frac{|\mathcal D|}{\mathit{df}(w,\mathcal D)} \right).
    \label{eq:tfidf}
\end{equation}
%\todo{How to get a small hyphen in tf-idf in math mode?}
%\todo{Should change all of the above (and below) to be (t,d) rather than (w,d)}
While the expression above captures our intuition that the term frequency should be high while the document frequency is relatively low, the specific expression may seem somewhat arbitrary (e.g.~the inclusion of the $\log_2$ term). Indeed, this expression is merely a heuristic, as described in the original implementation \cite{jones1972statistical}. Later work has attempted to justify this choice, e.g.~by interpreting $\log_2\frac{|\mathcal D|}{\mathit{df}(w,\mathcal D)}$ as a log-probability of a word appearing in a document \cite{robertson2004understanding}, though these are post-hoc justifications for what was ultimately a heuristic choice.

Likewise, the term frequency is also a heuristic and is often modified for use in specific contexts. For instance, two alternate definitions of the term frequency include:
\begin{eqnarray}
\mathit{tf}'(w,d) & = \delta(w \in d)\\
\mathit{tf}''(w,d) & = \frac{\mathit{tf}(w,d)}{\max_{w' \in d} \mathit{tf}(w',d)}.
%\todo{change w to t, maybe w' to w}
\end{eqnarray}
Both of the above are essentially normalization schemes, intended to prevent \emph{tf-idf} scores being higher for longer documents.

\subsection{Using TF-IDF for Search and Retrieval}

Cover BM-25?

% \section{Recurrent Models of Text}


\section{Distributed Word, Item, and Document Representations}
%\todo{Don't actually have document representations, add them or change the title}

\emph{Word2Vec} is a popular approach to developing \emph{semantic representations} of words \cite{mikolov2013distributed}. Such representations are somewhat analogous to the user and item representations ($\gamma_u$ and $\gamma_i$) we have been studying throughout this book. 
%That is, just as our user and item representations help us to predict which items `go with' which user (based on some latent representation of user preferences and item properties), word representations ought to tell us which words are likely to `go with,' or appear in the 
That is, just as our latent item representations give us a sense of which items are `similar to' which others (likewise for users), we would like to find latent word representations $\gamma_w$ that tell us which words are similar, or are likely to appear in the same context as each other.

These types of `distributed' word representations are potentially useful for a variety of reasons:
\begin{itemize}
    \item Unlike bag-of-words models (\Cref{sec:bagOfWords}), distributed representations offer a natural mechanism to handle synonyms. That is, words $w$ and $v$ that are synonyms of each other ought to have nearby representations $\gamma_w$ and $\gamma_v$, since they will tend to appear in the same contexts.
    \item Beyond synonyms, distributed representations might allow us better understand \emph{relationships} among words.
    %\todo{Say something more about this (king vs. queen etc.)
    \item In certain settings, distributed representations allow us to avoid dimensionality issues associated with bag-of-words models. For instance, when developing generative models of text (which we'll touch on briefly in \Cref{sec:generativeText}), documents are typically represented as sequences of low-dimensional word vectors $\gamma_w$, rather than as vectors of (e.g.)~word IDs via a bag-of-words model.
\end{itemize}

Below we briefly outline \emph{word2vec} as described in \cite{mikolov2013distributed}; in \Cref{sec:item2vec} we describe how this idea applies to learning item representations $\gamma_i$ for recommendation.
Although the latter and the former are essentially equivalent, the latter may feel more familiar, as it is similar to the way we learned item representations $\gamma_i$ using sequence models in \Cref{sec:?}.
%\todo{Is above useful? And does it reflect the final structure?}

Methodologically, \emph{word2vec} seeks to model the probability that a word in a sequence $w_t$ appears nearby words $w_{t+k}$, i.e.,~$p(w_t | w_{t+k})$. So, for a sequence of words $w_1 \ldots w_T$, we would like to learn word representations that maximize the (log) probability
\begin{equation}
    \frac{1}{T} \sum_{t=1}^T \sum_{-c \leq j \leq c, j \neq 0} \log p(w_{t+j} | w_t).
    \label{eq:word2vec0}
\end{equation}
%https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf
Here $c$ is the size of a \emph{context window}, which determines how many neighboring words we consider; this is a hyperparameter
%\todo{do I define this word somewhere?}
that may be chosen to balance accuracy and training time, though potentially can vary depending on the word $w_t$.
A simple way to define the probability $p(w | v)$ is to say that words $w$ are likely to appear near words with similar representations. In  \cite{mikolov2013distributed} this is defined in terms of the inner product between representations:
\begin{equation}
    p(w_o | w_i) = \frac{e^{\gamma'_{w_o} \cdot \gamma_{w_i}}}{\sum_{w \in \mathcal W} e^{\gamma'_w \cdot \gamma_{w_i}}}
    \label{eq:word2vec1}
\end{equation}
%\todo{Do I normally use \mathcal W for a dictionary or word set?}
where $\mathcal W$ is the set of words in the dictionary.
The numerator in the above encodes the compatibility between the `input' and `output' words $w_i$ and $w_o$; the denominator simply normalizes the value so that it corresponds to a probability (much as we saw in \Cref{sec:multiclassClassification}).
%\todo{Do I discuss multiclass anywhere?}

Note also that we learn \emph{two} representations, $\gamma_w$, and $\gamma'_w$ for each word (referred to as the `input' and `output' representation, respectively). 
Although doing so doubles the number of parameters, this type of representation avoids symmetries, for example a word is not likely to appear near itself. This is similar to the idea we saw when developing item-to-item recommender systems, and
sequential recommender systems in \Cref{sec:?},
%\todo{Do I ever describe such an example? Maybe in clothing?}
again the intuition being that an item is not likely to be co-purchased with itself.

Since the denominator in \Cref{eq:word2vec1} requires normalizing across all words in the dictionary $\mathcal W$, it is not efficient to compute. \cite{mikolov2013distributed} suggests a few schemes to overcome this issue, though the most straightforward is simply to sample a small number of `negative' words, rather than normalizing over the whole dictionary. As such each computation of $p(w_o | w_i)$ is replaced by an approximation:
\begin{equation}
    \log p(w_o | w_i) \simeq \sigma(\gamma'_{w_o} \cdot \gamma_{w_i}) + \sum_{w \in \mathcal N} \log \sigma(-\gamma'_{w} \cdot \gamma_{w_i})
    \label{eq:word2vec2}
\end{equation}
where $\mathcal N$ is a sampled set of negative words. \cite{mikolov2013distributed} proposes various schemes for choosing the sample $\mathcal N$, though most critically argues that the sampling probability should be proportional to the overall frequency of each word.

%\subsection{Doc2Vec}
%Probably skip, check e.g. https://medium.com/wisio/a-gentle-introduction-to-doc2vec-db3e8c0cce5e

\subsection{Item2Vec}
\label{sec:item2Vec}

\emph{Item2Vec}
\cite{barkan2016item2vec}
%ITEM2VEC: NEURAL ITEM EMBEDDING FOR COLLABORATIVE FILTERING (is this the actual item2vec paper?)
adapts the basic idea behind \emph{word2vec} as a means of learning item representations $\gamma_i$. The main difference between item2vec and word2vec is simply that sequences of words in sentences/documents are replaced by ordered sequences of items that each user has consumed. In practice this simply means that the probability in \Cref{eq:word2vec2} is replaced by
\begin{equation}
    \log p(i | j) \simeq \sigma(\gamma'_i \cdot \gamma_j) + \sum_{i' \in \mathcal N} \log \sigma(-\gamma'_{i'} \cdot \gamma_j),
\end{equation}
where $\mathcal N$ is a set of negative items, again sampled proportionally to overall item frequency.

\cite{barkan2016item2vec} discusses the effectiveness of this type of item representation in the setting of item-to-item recommendation on a corpus of song listens from \emph{Microsoft Xbox Music}. It was shown that the method naturally identifies latent dimensions that are associated with song genres; and it was argued qualitatively that related items are semantically more meaningful than those produced by alternate item-to-item recommendation techniques.

\subsection{Word2Vec and Item2Vec with Gensim}

Finally, we show how \emph{word2vec} and \emph{item2vec} work in practice, using interaction and review data from beer reviews, much as we did in \Cref{sec:?}.

To learn word representations, the input to the model is a list of documents (in this case reviews), each of which is a list of tokens (words). For this example we first remove capitalization and punctuation before tokenizing, as in \Cref{sec:?}.

Here we use the \emph{Gensim} implementation of \emph{word2vec}.\footnote{\url{https://radimrehurek.com/gensim/}}
%\todo{Don't know if I want urls floating around}
%\todo{Do I have one for that other library? If so reference consistently}
The model takes as input our tokenized reviews (in this case, we use a corpus of 50,000), a minimum word frequency, a dimensionality (i.e.,~$|\gamma_i|$), and a window size (i.e.,~$c$ in \Cref{eq:word2vec0}). The final argument specifies which specific version of the model is used, which corresponds to the model presented above:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from gensim.models import Word2Vec

model = Word2Vec(reviewTokens, # Tokenized documents
                 min_count=5, # Minimum frequency before words are discarded
                 size=10, # Model dimensionality
                 window=3, # Window size
                 sg=1) # Skip-gram model

model.wv.similar_by_word("grassy")
\end{lstlisting}
\end{minipage}

In the final line, we retrieve the most similar words for a particular query; in \emph{Gensim} this is based on the cosine similarity (\Cref{eq:cosinSim}) between the two word vectors:
\begin{equation}
    \max_{w} \frac{\gamma_w \cdot \gamma_{\mathit{grassy}}}{\| \gamma_w \| \| \gamma_{\mathit{grassy}} \|} = \text{`citrus'},
\end{equation}
followed by `citric', `floral', `flowery', `piney', `herbal', etc.
%, i.e.,~the words that maximize $\gamma'{w_o} \cdot \gamma_{w_i}$

Similarly, we can use the same code to run \emph{item2vec}, where our tokenized reviews are replaced by lists of items (i.e.,~product IDs) that each user has consumed (ordered by time).

After training a model on review histories, we find that the most similar beers to \emph{Molson Canadian Light} are other light beers such as \emph{Miller Light}, \emph{Molson Golden}, \emph{Piels}, \emph{Coors Extra Gold}, \emph{Labatt Canadian Ale}, (etc.) In \Cref{fig:item2vec} we train a two-dimensional item2vec model
%\todo{italics}
for the sake of visualizing the data, which reveals that beers belonging to different categories tend to occupy different parts of the item space.\footnote{A more effective visualization might be produced by using higher-dimensional embeddings, followed by a distance-preserving visualization technique like t-SNE \cite{maaten2008visualizing}, though for here we plot the embedding dimensions directly for simplicity.}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/item2vec.pdf}
\caption{Item representations ($\gamma_i$) using a two-dimensional \emph{item2vec} model. Representations for items from three distinct categories are shown. \label{fig:item2vec}}
\end{SCfigure}



\section{Personalized Sentiment and Recommendation}

The models for text we have explored so far, although they can be applied to data like reviews, and can be used to recommend related documents, are ultimately not personalized.

Several attempts have been made to combine models of text with models of users and preferences, and in particular with recommendation approaches. For example, just as we saw techniques in \Cref{chap:content} that improve recommender systems through the use of side-information, \emph{text} can be useful to efficiently understand the dimensions of user's opinions.

Often there is a significant amount of textual data available in addition to interactions that might be leveraged to fit better models. Other than helping understand sentiment, text can help to understand the dimensions of items and preferences, e.g.~the different properties of products and the different aspects that users care about.

Text can also be helpful for model \emph{interpretability}. So far, the recommender systems we've developed are essentially `black boxes,' whose predicions (as in \Cref{eq:latentFactor?}) are defined purely in term of latent factors. Models of text can be used to understand what aspects these latent dimensions correspond to, to summarize reviews, or to explain recommendations.
%\todo{Add references if I eventually have sections on these things}

However, extracting \emph{meaningful} information from text is challenging. E.g.~most of the simple text representations we've seen so far (\Cref{sec:bagOfWords}) are high-dimensional and not particularly sparse; simply incorporating such features into general-purpose feature-aware models is possibly not effective, and one must instead design methods to work specifically with text. Below we cover a few representative approaches.

%TODO: Anything about personalized sentiment analysis specifically?

\subsection{Case Studies: Review-Aware Recommendation}
\label{sec:reviewAware}

\emph{Product reviews} are often used as a source of information to improve recommendation performance. Conceptually, reviews ought to tell us much more about preferences and opinions than (e.g.)~a single rating can. This could be especially true for latent dimensions ($\gamma_u$ and $\gamma_i$ for users and items) since product reviews are intended to `explian' opinions the underlying dimensions behind users' decisions.

Roughly speaking, there are two schools of thought as to how reviews should be incorporated to improve recommendation performance. One option is to treat review text as a form of \emph{regularization}, essentially to encourage the low-dimensional representations of users or items (via $\gamma_u$ and $\gamma_i$) to be similar to low-dimensional representations extracted from text. Others seek to extract representations from text that can be used to improve feature-based matrix factorization methods. We give examples of both below.

\paragraph{Hidden Factors as Topics}
%\todo{probably rearrange so that this is a subsection}

An early attempt to incorporate text into recommender systems attempted to do so by making use of \emph{topic models} applied to product reviews \cite{mcauley2013hidden}.
%
Topic models \cite{blei2003latent} learn low-dimensional representations of text (essentially finding structure in the types of bag-of-words representations we covered in \Cref{sec:bagOfWords}). `Topics' then correspond to sets or clusters of words that tend to co-occur together. For example, a topic model trained on movie reviews might dicsover that groups of words associated with `action,' `comedy,' or `romance,' might tend to co-occur together, and therefore that these are distinct topics among movie reviews (we give examples of actual topics in \Cref{fig:?}).
%\todo{Separate topic on dimensionality reduction in text?}

The basic idea in \cite{mcauley2013hidden} is that the low dimensional structure among \emph{reviews} should be related to the low-dimensional structure in \emph{ratings}---after all, reviews are intended to explain why a user rated a product a certain way. Furthermore, even though a single rating can tell us very little about the underlying dimensions that explain a user's ratings, a single review potentially contains enough information to 

The method is somewhat reminiscent of the social recommendation approach we covered in \Cref{sec:social?} \cite{??}, in which a shared parameter $\gamma_u$ was tasked with simultaneously explaining rating dimensions as well as social connections. The argument we made in \Cref{sec:social?} was that in the absence of sufficient rating data, our estimate of $\gamma_u$ will then be estimated from $u$'s friends.

Likewise, \cite{mcauley2013hidden} suggested that a shared parameter $\gamma_u$ could simultaneously explain rating dimensions via a latent factor model, as well as the topics in reviews:
\begin{equation}
    \underbrace{\sum_{(u,i) \in \mathcal T} (\alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i - r_{u,i})^2}_{\text{rating error}} + \lambda \underbrace{\sum_{(u,i) \in \mathcal T} \sum_{w \in d_{u,i}} \log p(w | \gamma_u, \psi)}_{\text{topic likelihood}},
\end{equation}
%\todo{(u,i) \in or u,i \in ?}
%\todo{equation renders poorly}
where $\psi$ is a set of additional (non-shared) parameters specific to the topic model (much like the approach in \Cref{sec:social?} had shared and non-shared social parameters).

Critically, the above model assumes an alignment between latent rating dimensions and review topics. In practice, there may be dimensions in reviews (i.e.,~topics) which are not related to rating dimensions (e.g.~if a user discusses the plot in a book review, it may have little connection to their rating); likewise there may be `intangible' latent dimensions that don't correspond to topics expressed in reviews. By assuming a one-to-one relationship between topics and latent dimensions, the model is useful in cold- (or cool-) start settings by forcing the topic model to discover \emph{those topics that are capable of explaining the variation in ratings}. These topics in particular will be the ones that help us to quickly understand the dimensions that explain user ratings from a few interactions.
%\todo{wordy}

Examples of the types of topics discovered by this model are shown in \Cref{fig:hftTopics}; mostly, the discovered factors correspond to fine-grained product categories, which mostly reflect users' tendency to favor certain types of item over others.

\begin{figure}
    \centering
\begin{tikzpicture}
 %\draw[dashed] (0,0) rectangle (5,4) node[pos=.5] (A) {Dataset $\lbrace X; y \rbrace$};
% \node [below=of A,yshift=0em] {Estimate $\theta$};
\node [style=circle,draw] (gamma1) {$\gamma_i$};
\node [right=of gamma1, circle, fill, inner sep=0.1em] (pt1) {};
\node [style=circle,draw,right=of pt1] (gamma2) {$\gamma_u$};
\node [right=of gamma2, circle, fill, inner sep=0.1em] (pt2) {};
\node [style=circle,draw,right=of pt2] (psi) {$\psi$};
\node [below=of pt1] (out1) {rating};
\node [below=of pt2] (out2) {word frequency};
\draw[-stealth] (pt1) -- (out1);
\draw[-stealth] (pt2) -- (out2);
\draw (gamma1) -- (pt1);
\draw (pt1) -- (gamma2);
\draw (gamma2) -- (pt2);
\draw (pt2) -- (psi);

\node[style=dashed,draw, inner sep=3mm, yshift=0.2em, fit=(gamma1) (gamma2)] (Fit1) {};
\node[style=dashed,draw, inner sep=3mm, yshift=-0.2em, fit=(gamma2) (psi)] (Fit2) {};

\node[above=of Fit1,yshift=-3em] (label1) {ratings};
\node[above=of Fit2,yshift=-2.6em] (label2) {document topics};

\end{tikzpicture}
    \caption{Similar to the social recommendation models from \Cref{sec:?}, personalized models of text often make use of a \emph{shared} parameter that must simultaneously explain structure in interactions and documents.}
    \label{fig:hftIdea}
    %\todo{Reference this somewhere!}
\end{figure}

\paragraph{Other Topic-Modeling Approaches}

The above is a simple approach to combine low-dimensional representations of text (via a topic model) with low-dimensional representations of interactions (via a latent factor model).
%\todo{Do I have the above sentence already somewhere?}
Several others have adopted a similar approach, typically by modifying how user factors $\gamma_u$ and topic dimensions are related to each other.

\cite{ling2014ratings} and \cite{diao2014jointly} both consider the same setting as above, in which reviews are used to improve the performance of rating prediction. Both note the limitations of assuming a simple one-to-one mapping between review topics and user preferences as in \cite{mcauley2013hidden}, and 

\cite{wang2011collaborative} proposed a similar approach to the problem of recommending scientific articles, where document representations are extracted from article text, and user preferences are modeled to predict which articles they will include in their libraries. This differs from the above formulation in that text is associated with items (documents) rather than users, and the setting is essentially an instance of `one-class' recommendation (as in \Cref{sec:oneclassRec}), since one generally doesn't have explicit negative feedback about the articles a user didn't read.



\paragraph{Neural-Network Approaches}

\cite{zheng2017joint} CNN based approach, treat user and item reviews as a document.

\begin{figure}
    \centering
    {
    \small
    \begin{tabular}{cccccccccc}
    % \toprule
    theaters & spas & mexican & vietnamese & snacks & italian & medical & donuts & coffee & seafood\\
    % \midrule
theater & massage & mexican & pho & cupcakes & pizza & dr & donuts & coffee & sushi\\
movie & spa & salsa & vietnamese & cupcake & crust & stadium & donut & starbucks & dish\\
harkins & yoga & tacos & yogurt & hotel & pizzas & dentist & museum & books & restaurant\\
theaters & classes & chicken & brisket & resort & italian & doctor & target & latte & rolls\\
theatre & pedicure & burrito & beer & rooms & bianco & insurance & subs & bowling & server\\
movies & trail & beans & peaks & dog & pizzeria & doctors & sub & lux & shrimp\\
dance & studio & taco & mojo & dogs & wings & dental & dunkin & library & dishes\\
popcorn & gym & burger & shoes & frosting & pasta & appointment & frys & espresso & menu\\
tickets & hike & carne & froyo & bagel & mozzarella & exam & tour & stores & waiter\\
flight & nails & food & zoo & bagels & pepperoni & prescription & bike & gelato & crab
% \bottomrule
    \end{tabular}
    }
    %\includegraphics{}
    \caption[Example of topics that explain variance in rating dimensions on \emph{Yelp}.]
    {
    Example of topics that explain variance in rating dimensions on \emph{Yelp}, from \cite{mcauley2013hidden}.
    }
    \label{fig:hftTopics}
\end{figure}
%\todo{Should be a figure or a table? Depends if I have enough tables for it to be worth having an index for them}
%\todo{Fix table formatting}
%\todo{Move figure to where it is referenced}





% \paragraph{How Useful are Reviews in Practice?}

\begin{figure}
\caption{How Useful are Reviews for Recommendation?}
Although user reviews have been used in several academic studies of text-based recommendation, it is worth thinking about how useful such data would be in practice. After all, in real-world settings the vast majority of interactions would not be associated with reviews. \cite{sachdeva2020useful} studied the settings
\label{box:usefulReviews}
%\todo{referenced anywhere?}
\end{figure}


\section{Question Answering Systems}

\section{Personalized Text Generation}

In \Cref{sec:lstm} we presented Recurrent Neural Networks as general-purpose models that can be used to estimate the next value in a sequence, or to generate sequences, based on some context and the sequence of tokens seen so far.

Such models are routinely used to sample (or generate) realistic-looking text. Recurrent networks for text generation (see e.g.~\cite{graves2013generating})
%\todo{good reference? see others from jianmo paper}
follow essentially the same setup we saw in \Cref{sec:rnn?}. 
%where characters (or words) are generated by the model, following h
At each step $t$, the network takes 
an input $x_t$ (either a character, or a word representation),
%for each step t and 
and
updates its hidden states $h_t$
based on the current input $x_t$
and the previous step's hidden state
$h_{t-1}$ (as in \Cref{sec:rnn?}, multiple network layers can be stacked. 
The sequence of target outputs $y$ is identical to $x$, but shifted by a single token, i.e.,~the model is responsible for generating the \emph{next} token in a sequence based on all the tokens seen so far.
%The model is trained autoregressively, i.e.,~each network outputs a token $y_t$, which is then 
This type of setup is depicted in \Cref{fig:?}.
%\todo{Is this same notation from RNN section?}
%\todo{Is text distinct }
%t−1
%The network predicts the input $x_{t+1}$ at the
%next step, given all inputs $x_{\leq t}$.
%The output layer is connected to a softmax laye
%\done{looks okay, based on what Jianmo wrote}
%This is the essentially the same setting as we described when introducing recurrent networks as general-purpose sequence models in \Cref{chap:temporalSequential} (\Cref{sec:?}).

While the above model will learn to generate realistic-looking samples (i.e.,~documents that mimic those in the training set), the samples will not be context-dependent, and they will not be \emph{personalized}.

Several papers have sought to adapt RNNs to generate \emph{personalized} text, that is, text that mimics the context, preferences, or writing style of an individual user.
We describe several such approaches below. Most are models of product reviews: partly because such data exhibits variation due to the user, item, and interaction between them, but also simply because such data is widely available. 
We discuss the broader value of this type of technique 
%after presenting several models below.
below.
%\todo{clunky text, could move explanation text to here or give it a subsection heading to reference it}

%Several papers have focused on learning language models to generate coherent reviews.
%\done{More introducory text}

%\todo{repetitive (above and below para}

\paragraph{Why Generate Reviews?}
%\done{Moved to top}
Personalized text generation, especially when applied to datasets of product reviews, may seem an unusual task with no obvious application (beyond generating review spam): existing platforms are unlikely to surface generated reviews (such as the one in \Cref{fig:?}) to users. However this task has more value when considering the broader context of personalized language generation. In practice, as we've seen elsewhere, reviews are often simply a convenient test-bed for training due to the availability of data. Other applications where personalized language generation could include dialog systems (\Cref{sec:?}), assistive language tools, or natural language processing in other datasets with significant personal variability (e.g.~clinical NLP).
%\done{is test-bed a word? is it used elsewhere? Who cares}
%

Even within the context of reviews, a high-fidelity personalized language model can be used for other functions besides generation. For example the model could be `reversed' for personalized retrieval or search, i.e.,~to retrieve items \emph{that a user might describe in a particular way} (e.g.~a query such as `good dress for summer wedding').
%\todo{writing not totally clear}
Personalized text generation can also be used within systems that explain or justify predictions, as we see in 
%the following section.
\Cref{sec:?}.
%\todo{make sure it's still the following section}

% Other applications besides reviews, health, personalized writing
% Justification generation



\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Summary of personalized text generation approaches.}
\label{tab:personTextGen}
%\todo{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{|XXX|}
\hline
\textbf{Goal} & \textbf{Reference} & \textbf{Description}\\
Sentiment analysis & \cite{radford2017learning} & Focuses on the task of generating user reviews, though is not personalized; the goal is mainly to learn disentangled representations and discover sentiment.\\
Attribute-based generation & \cite{dong2017learning} & An encoder-decoder approach to generate reviews based on contextual attributes (such as the user, item, or rating). \\
Abstractive tip generation & \cite{li2017neural} & Generates short `tips' (similar to review summaries), also based on an encoder-decoder approach; reviews are used during training to learn an intermediate representation. \\
Personalized review generation & \cite{ni2017estimating} & Generates reviews using a latent-factor approach that takes user and item factors as input. \\ 
\hline
\end{tabularx}
\end{minipage}
\end{table}
%\todo{Could add justification approaches}

\begin{figure}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]
%\todo{>=LaTeX above controls the arrow style. Is it good?}
%  \begin{scope}[start chain=going right,nodes=itemc,every
%  join/.style={-latex,very thick},local bounding box=chain]
%  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
%  {$A$};
%  \end{scope}
%  \node[left=1em of chain,scale=2] (eq) {$=$};


 \node[item, minimum width=5mm] (x11) {};
 \node[item, right=of x11, xshift=-10pt, minimum width=5mm] (x12) {};
 \node[item, right=of x12, xshift=-10pt, minimum width=5mm] (x13) {};
 \node[item, right=of x13, xshift=-10pt, minimum width=5mm] (x14) {};
 \node[item, right=of x14, xshift=-10pt, minimum width=5mm] (x15) {};
 \node[item, right=of x15, xshift=-10pt, minimum width=5mm] (x16) {};
 \node[item, right=of x16, xshift=-10pt, minimum width=5mm] (x17) {};
 \node[item, right=of x17, xshift=-10pt, minimum width=5mm] (x18) {};
 \node[right=of x18, xshift=-10pt, minimum width=5mm] (x19) {$\ldots$};

 
 \node[item, below=of x11, yshift=8pt,minimum width=5mm,rectangle,fill=gray!10] (x01) {\texttt{\textless{}s\textgreater{}}};
 \node[item, below=of x12, yshift=10pt, minimum width=5mm,fill=gray!10] (x02) {I};

 \node[item, below=of x13, yshift=10pt, minimum width=5mm,fill=gray!10] (x03) {\_};
 \node[item, below=of x14, yshift=10pt, minimum width=5mm,fill=gray!10] (x04) {w};
 \node[item, below=of x15, yshift=10pt, minimum width=5mm,fill=gray!10] (x05) {a};
 \node[item, below=of x16, yshift=10pt, minimum width=5mm,fill=gray!10] (x06) {s};
 \node[item, below=of x17, yshift=10pt, minimum width=5mm,fill=gray!10] (x07) {\_};
 \node[item, below=of x18, yshift=10pt, minimum width=5mm,fill=gray!10] (x08) {d};

 
%  \node[item, above=of x11, minimum width=5mm] (x21) {};
%  \node[left=of x21, xshift=10pt] (x20) {$h^{l}_{t-2}$};
%  \node[item, above=of x12, minimum width=5mm] (x22) {};
%  \node[item, above=of x13, minimum width=5mm] (x23) {};
%  \node[item, above=of x14, minimum width=5mm] (x24) {};
%  \node[right=of x24, xshift=-10pt] (x25) {$h^{l}_{t+2}$};

 \node[item, above=of x11, yshift=-10pt, minimum width=5mm,fill=gray!10] (x31) {I};
 \node[item, above=of x12, yshift=-10pt, minimum width=5mm,fill=gray!10] (x32) {\_};
 \node[item, above=of x13, yshift=-10pt, minimum width=5mm,fill=gray!10] (x33) {w};
 \node[item, above=of x14, yshift=-10pt, minimum width=5mm,fill=gray!10] (x34) {a};
 \node[item, above=of x15, yshift=-10pt, minimum width=5mm,fill=gray!10] (x35) {s};
 \node[item, above=of x16, yshift=-10pt, minimum width=5mm,fill=gray!10] (x36) {\_};
 \node[item, above=of x17, yshift=-10pt, minimum width=5mm,fill=gray!10] (x37) {d};
 \node[item, above=of x18, yshift=-10pt, minimum width=5mm,fill=gray!10] (x38) {e};
 

%  \draw [->] (x11) -- node[right] {$h^{l-1}_{t-1}$} (x21);
%  \draw [->] (x12) -- node[right] {$h^{l-1}_{t}$} (x22);
%  \draw [->] (x13) -- node[right] {$h^{l-1}_{t+1}$} (x23);
%  \draw [->] (x14) -- node[right] {$h^{l-1}_{t+2}$} (x24);
 
 \draw [->] (x11) -- (x12);
 \draw [->] (x12) -- (x13);
 \draw [->] (x13) -- (x14);
 \draw [->] (x14) -- (x15);
 \draw [->] (x15) -- (x16);
 \draw [->] (x16) -- (x17);
 \draw [->] (x17) -- (x18);
 \draw [->] (x18) -- (x19);
 
%  \draw [->] (x21) -- node[above] {$h^{l}_{t-1}$} (x22);
%  \draw [->] (x22) -- node[above] {$h^{l}_{t}$} (x23);
%  \draw [->] (x23) -- node[above] {$h^{l}_{t+1}$} (x24);
%  \draw [->] (x24) -- node[right] {} (x25);
 
%  \draw [->] (x10) -- (x11);
%  \draw [->] (x20) -- (x21);
 
 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);
 \draw [->] (x05) -- (x15);
 \draw [->] (x06) -- (x16);
 \draw [->] (x07) -- (x17);
 \draw [->] (x08) -- (x18);
 
 \draw [->] (x11) -- (x31);
 \draw [->] (x12) -- (x32);
 \draw [->] (x13) -- (x33);
 \draw [->] (x14) -- (x34);
 \draw [->] (x15) -- (x35);
 \draw [->] (x16) -- (x36);
 \draw [->] (x17) -- (x37);
 \draw [->] (x18) -- (x38);
\end{tikzpicture}
%OLD
% \begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
% itemc/.style={item,on chain,join}]
% %  \begin{scope}[start chain=going right,nodes=itemc,every
% %  join/.style={-latex,very thick},local bounding box=chain]
% %  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
% %  {$A$};
% %  \end{scope}
% %  \node[left=1em of chain,scale=2] (eq) {$=$};
%  \node[item] (AL1) {$A$};
%  \node[item, right=of AL1, xshift=-1em] (AL2) {$A$};
 
% %  \path (AL.west) ++ (-1em,2em) coordinate (aux);
% %  \draw[very thick,-latex,rounded corners] (AL.east) -| ++ (1em,2em) -- (aux) 
% %  |- (AL.west);
% %  \foreach \X in {0,1,2,t} 
% %  {\draw[very thick,-latex] (A\X.north) -- ++ (0,2em)
% %  node[above,item,fill=gray!10] (h\X) {$h_\X$};
% %  \draw[very thick,latex-] (A\X.south) -- ++ (0,-2em)
% %  node[below,item,fill=gray!10] (x\X) {$x_\X$};}
% %  \draw[white,line width=0.8ex] (AL.north) -- ++ (0,1.9em);
%  \draw[thick,-latex] (AL1.north) -- ++ (0,2em) node[above,item,fill=gray!10] {$h_t$};
%  \draw[thick,latex-] (AL1.south) -- ++ (0,-2em) node[below,item,fill=gray!10] {$x_t$};
% %  \path (x2) -- (xt) node[midway,scale=2,font=\bfseries] {\dots};
% \end{tikzpicture}
\caption{Recurrent neural network for text generation. At each step the network is responsible for generating the next token (in this case a character) on the basis of the tokens seen so far, and the network's current hidden state. `\textless{}s\textgreater{}' represents a `start token' to begin generation (and generation would be terminated once the network generates an end token `\textless{}\textbackslash{}s\textgreater{}'). \label{fig:rnnText}}
%\todo{figure not referenced?}
\end{figure}





\begin{figure}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]


 \node[item, minimum width=5mm] (x11_) {};
 \node[item, right=of x11, xshift=-15pt, minimum width=5mm] (x12_) {};
 \node[item, right=of x12, xshift=-20pt, minimum width=5mm] (x13_) {};
 \node[item, right=of x13, xshift=-25pt, minimum width=5mm] (x14_) {};
 \node[right=of x14, xshift=-30pt, minimum width=5mm] (x15_) {$\ldots$};
 
 \node[item, below=of x11, yshift=8pt,minimum width=5mm,rectangle] (x01_) {$\gamma_u;\gamma_i$};
 \node[item, below=of x12, xshift=-5pt,yshift=10pt, minimum width=5mm,fill=gray!10] (x02_) {I};
 \node[item, below=of x13, xshift=-10pt, yshift=10pt, minimum width=5mm,fill=gray!10] (x03_) {\_};
 \node[item, below=of x14, xshift=-15pt, yshift=10pt, minimum width=5mm,fill=gray!10] (x04_) {w};

 \node[item, above=of x11, yshift=-10pt, minimum width=5mm,fill=gray!10] (x31_) {I};
 \node[item, above=of x12, xshift=-5pt, yshift=-10pt, minimum width=5mm,fill=gray!10] (x32_) {\_};
 \node[item, above=of x13, xshift=-10pt, yshift=-10pt, minimum width=5mm,fill=gray!10] (x33_) {w};
 \node[item, above=of x14, xshift=-15pt, yshift=-10pt, minimum width=5mm,fill=gray!10] (x34_) {a};

 \draw [->] (x11_) -- (x12_);
 \draw [->] (x12_) -- (x13_);
 \draw [->] (x13_) -- (x14_);
 \draw [->] (x14_) -- (x15_);

 \draw [->] (x01_) -- (x11_);
 \draw [->] (x02_) -- (x12_);
 \draw [->] (x03_) -- (x13_);
 \draw [->] (x04_) -- (x14_);

 \draw [->] (x11_) -- (x31_);
 \draw [->] (x12_) -- (x32_);
 \draw [->] (x13_) -- (x33_);
 \draw [->] (x14_) -- (x34_);


 \node[item, minimum width=5mm, right=of x11_, xshift=100pt] (x11) {};
 \node[item, right=of x11, xshift=-15pt, minimum width=5mm] (x12) {};
 \node[item, right=of x12, xshift=-15pt, minimum width=5mm] (x13) {};
 \node[item, right=of x13, xshift=-15pt, minimum width=5mm] (x14) {};
 \node[right=of x14, xshift=-15pt, minimum width=5mm] (x15) {$\ldots$};
 
 \node[item, below=of x11, yshift=8pt,minimum width=5mm,rectangle,fill=gray!10] (x01) {\texttt{\textless{}s\textgreater{}}};
 \node[item, below=of x12, yshift=10pt, minimum width=5mm,fill=gray!10] (x02) {I};
 \node[item, below=of x13, yshift=10pt, minimum width=5mm,fill=gray!10] (x03) {\_};
 \node[item, below=of x14, yshift=10pt, minimum width=5mm,fill=gray!10] (x04) {w};

 \node[below=of x01, yshift=8pt,minimum width=5mm,rectangle] (g01) {$\gamma_u;\gamma_i$};
 \node[below=of x02, yshift=10pt, minimum width=5mm] (g02) {$\gamma_u;\gamma_i$};
 \node[below=of x03, yshift=10pt, minimum width=5mm] (g03) {$\gamma_u;\gamma_i$};
 \node[below=of x04, yshift=10pt, minimum width=5mm] (g04) {$\gamma_u;\gamma_i$};

 \node[item, above=of x11, yshift=-10pt, minimum width=5mm,fill=gray!10] (x31) {I};
 \node[item, above=of x12, yshift=-10pt, minimum width=5mm,fill=gray!10] (x32) {\_};
 \node[item, above=of x13, yshift=-10pt, minimum width=5mm,fill=gray!10] (x33) {w};
 \node[item, above=of x14, yshift=-10pt, minimum width=5mm,fill=gray!10] (x34) {a};

 \draw [->] (x11) -- (x12);
 \draw [->] (x12) -- (x13);
 \draw [->] (x13) -- (x14);
 \draw [->] (x14) -- (x15);

 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);

 \draw [->] (g01) to [out=45,in=315] (x11);
 \draw [->] (g02) to [out=45,in=315] (x12);
 \draw [->] (g03) to [out=45,in=315] (x13);
 \draw [->] (g04) to [out=45,in=315] (x14);

 \draw [->] (x11) -- (x31);
 \draw [->] (x12) -- (x32);
 \draw [->] (x13) -- (x33);
 \draw [->] (x14) -- (x34);

\end{tikzpicture}
\caption{Personalized (or contextual) recurrent network architectures. Left: an encoder-decoder architecture; here the start token from \Cref{fig:rnnText1} is replaced by an input signal produced by (or jointly trained with) a previous model (in this case encoding user and item information). Right: the generative-concatenative network from \cite{?}; here the contextual information is input during every step to help the model `remember' the context over many steps.
\label{fig:rnnText2}}
%\todo{figure not referenced, also put somewhere}
\end{figure}



\paragraph{RNN-based Review Generation}
%
\cite{radford2017learning} was among the first papers to explore the use of recurrent neural networks (and in particular, LSTM neural networks, as in \Cref{sec:LSTM}) to generate realistic-looking reviews.

Although the model from \cite{radford2017learning} is not personalized, the approach shows the effectiveness of recurrent networks to sample realistic reviews; in particular, the model operates at the \emph{character} level (i.e.,~the sequences as in \Cref{fig:lstm?} are 

\paragraph{Conditional Review Generation}
%
Given the promise of using recurrent neural networks to sample realistic-looking reviews from a background distribution, several papers have followed a similar approach to \cite{radford2017learning} to generate reviews that are relevant to a specific context.

Conceptually, generating reviews to match a specific context follows ideas from \emph{encoder-decoder} architectures which have proved useful in (for example) image captioning settings \cite{vinyals2015show}. Here, rather than passing a start token to the generator, as in \Cref{fig:CharacterLSTMdemo}, one passes an encoding of (i.e.,~a low dimensional representation) the context, such as an embedding representing an image; following this, decoding follows the same approach as in \Cref{fig:CharacterLSTMdemo},
%\todo{is this the right figure?}
where the model's hidden states ought to retain the essential components of the context necessary to generate text conditionally.
%\todo{clunky text}
%\todo{maybe make a conditional generation diagram (encoder decoder) and refer to it above}


%\cite{dong2017learning} (attribute based generation)
%\todo{This method belongs in personalized review generation, since it is personalized. move below}
\cite{dong2017learning} follow this type of encoder-decoder approach to accomplish `attribute-based' conditional review generation. The approach follows the setting described above, whereby attributes are encoded and passed to an LSTM text generation model. The attributes used in their model include a user ID, an item ID, and the score associated with the review.\footnote{Though this dependency on having the rating as an input could presumably be overcome by estimating it separately.}
%\todo{Say something about results}

\cite{li2017neural} follow a similar approach to generate short `tips,' i.e.,~short summaries of reviews. The setting also follows an encoder-decoder approach, though is trained on data from \emph{Yelp} and \emph{Amazon} that includes both reviews and summaries (or `tips' on Yelp).
%\todo{italics}
While summaries are used as the model output, reviews are used during training to learn an effective intermediate representation that explains interactions between users and items.
%\todo{maybe move below as a segue to explanation generation}
%(abstractive tip generation)

\paragraph{Personalized Review Generation}
%
The above methods generate reviews based on specific features or attributes, and as such can essentially be thought of as forms of `contextual' personalization as we discussed in \Cref{sec:?}.
\cite{ni2017estimating} designed text generation methods that directly model users (and items) in order to estimate reviews given only the context of a user and item ID.

The basic setup follows a latent factor approach, i.e.,~the `input' to the model is a representation $\gamma_u$ for the user and $\gamma_i$ for the item. These latent user and item representations are trained jointly with the language model (in this case an LSTM, as in \Cref{sec:?}); in practice these representations are concatenated onto the input tokens $x$ as in \Cref{fig:?}.
%\todo{check notation}
Conceptually, rather than the latent factors $\gamma_u$ and $\gamma_i$ explaining user preferences and item properties that predict ratings (as in \Cref{sec:?}), user factors must now account for patterns of variation in user writing styles (e.g.~the structure used in their reviews), and item factors must account for the overall characteristics of items (e.g.~their objective properties) that users are likely to write about. At the same time both the user and item factors must \emph{jointly} explain the user's sentiment toward the item, e.g.~the positive or negative language that they will use.

An example of a review generated via this technique is shown in \Cref{fig:genConcat}. The review (in this case of a beer from \emph{beeradvocate})
%\todo{italics}
is surprisingly coherent and seems to capture (a) the user's writing style (e.g.~they tend to write their reviews across separate paragraphs describing each aspect); (b) the item's overall characteristics (e.g.~the category and flavor profile); and (c) the essential features of the user's preference (i.e.,~a lukewarm review). Recall that as with a traditional recommender system although both the user and item have been seen during training, though this specific user and item \emph{combination} have not.

\begin{figure}
\parbox{0.52\textwidth}{12 oz.~bottle, excited to see a new Victory product around, \textbf{A:}
Pours a dark brown, much darker than I thought it would be, rich
creamy head, with light lace. \textbf{S:} Dark cedar/pine nose with some
dark bread/pumpernickel. \textbf{T:} This ale certainly has a lot of malt,
bordering on Barleywine. Molasses, sweet maple with a clear
bitter melon/white grapefruit hop flavour. Not a lot of complexity
in the hops here for me. Booze is noticable. \textbf{M:} Full-bodied,
creamy, resinous, nicely done. \textbf{D:} A good beer, it isn't exactly
what I was expecting. In the end above average, though I found
it monotonous at times, hence the 3. A sipper for sure.}\hspace{0.04\textwidth}\parbox{0.44\textwidth}{\textbf{A:} Pours a very dark brown with a nice finger of tan
head that produces a small bubble and leaves decent
lacing on the glass. \textbf{S:} Smells like a nut brown ale.
It has a slight sweetness and a bit of a woody note
and a little cocoa. The nose is rather malty with some
chocolate and coffee. The taste is strong but not overwhelmingly sweet. The sweetness is overpowering,
but not overwhelming and is a pretty strong bitter finish. \textbf{M:} Medium bodied with a slightly thin feel. \textbf{D:}
A good tasting beer. Not bad.}
\caption{ \label{fig:genConcat}}
\end{figure}
%\todo{add highlights, and maybe select a different review}
%\todo{write caption}

Extensions of this work combined latent user and item representations with observed attributes \cite{ni2018personalized}. Doing so can help the model to use language that better captures specific item details (such as technical features of an electronics product). Further applications are proposed that use this type of technique for \emph{assistive} review generation, e.g.~rather than generating reviews from scratch, the system could be used to help users write reviews based on a template or specific points or attributes, while still following their personalized writing style.

%

\subsection{Text-Based Explanations and Justifications}

So far, we have examined (a) textual data as a means of improving the \emph{predictive accuracy} of personalized models (\Cref{sec:?}); and (b) textual data as the \emph{output} of a predictive model (\Cref{sec:?}). Beyond these applications, text is also appealing as a means of \emph{explaining} model predictions.

Text-based explanation connects to the more broad topics of \emph{interpretability}, \emph{explainability}, and \emph{justification} of machine learning models. In the case of text-based explanations, the goal is to retrieve or generate a short text fragment that explains 
a model prediction.
%Conceptually, 
%\done{explain how this is related to "human" notions of explanation}
Such models bring us a little closer to the conceptual goal of producing `human-like' explanations, i.e.,~mimicking the way one person would justify a decision or recommendation to another.

As with the personalized text generation models above, much of the work in this space is focused on recommendations and review datasets, as review data serves as a convenient testbed to train models for personalized explanation. The more broad topic of (non-personalized) text-based explanation has been considered in other settings, such as text-based classification
\cite{liu2018towards}.
%\todo{check}

\paragraph{Extractive vs.~Abstractive Approaches}
%(2) justifications (extractive, e.g.~Diego's paper)
Justification (and summarization) approaches can broadly be categorized in \emph{extractive} or \emph{abstractive}. `Extractive' approaches use retrieval-like approaches to select text fragments (e.g.~from a training corpus) that are relevant to a given context or query; `abstractive' approaches generate novel text, either by paraphrasing the corpus or via a generative approach (as in \Cref{sec:?}).
%\todo{Could drop extractive vs abstractive, not relevant
%Or Maybe make this an explanation box thing and reference somewhere
%}


\paragraph{Crowd-sourced Explanations}
%Crowd-Based Personalized Natural Language
%Explanations for Recommendations
Prior to the use of complex generative models for text-based explanation, \cite{chang2016crowd}
%https://dl.acm.org/doi/pdf/10.1145/2959100.2959153
sought to use \emph{crowd-sourcing} to generate personalized explanations for movie recommendations. Since crowd workers are unlikely to be available to generate recommendations in real-time, \cite{chang2016crowd} adopt a template-based approach, where workers generate justifications based on users' interest in a particular aspect or tag
(e.g.~why should \emph{Goodfellas} be recommended to a user who likes \emph{drama}?). To help crowd-workers write explanations, workers are shown extracted review segments relevant to a particular tag, and are tasked with abstracting that text into a coherent explanation.\footnote{The complete pipeline from \cite{chang2016crowd} includes a few additional details, such as a step to vote on the best among several crowd-based explanations, among others.}

Regardless of the practicality of crowd-sourcing such explanations, a main goal in \cite{chang2016crowd} is to confirm the overall value of text-based explanations to users (where human-generated explanations can be regarded as something of a gold-standard). They evaluate text-based explanations in terms of \emph{efficiency} (acquainting users with the relevant properties of an item), \text{effectiveness} (helping users decide whether they want to watch a movie), \emph{trust}, and \emph{satisfaction}.
They find that text-based explanations are preferred over more trivial tag-based explanations in terms of efficiency, trust, and satisfaction (though with negligible change in terms of effectiveness).




\paragraph{?}
Let Me Explain: Impact of Personal and Impersonal
Explanations on Trust in Recommender Systems \cite{kunkel2019let}

%\paragraph{Abstractive Approaches}
\paragraph{Generating Explanations from Reviews}
%(1) Text-based justifications (abstractive)
\cite{ni2019justifying} explores abstractive approaches to justification generation. Overall, the model is similar to the one described in \Cref{sec:?}, where the model is trained to generate a review given a particular user/item pair.
%\todo{style (user/item)}
\cite{ni2019justifying} uses a similar training setup, except that the target text used for training is text that has been identified as being suitable to surface as a recommendation justification. Ultimately this text is harvested from reviews, with a main challenge being how to identify suitable justification sentences among review text. \cite{ni2019justifying} argues that training on this type of harvested text yields more effective justifications than models trained on reviews, tips, or retrieval-based techniques.

Examples of justifications generated using different techniques and training setups (from \cite{ni2019justifying}) are shown in \Cref{fig:justification}.

\begin{figure}
\begin{tabularx}{\linewidth}{lll}
Method & Type & Output\\
Personalized Retrieval & Extractive & \emph{A great burger and fries}\\
Generated `tip' & Abstractive & \emph{This place is awesome}\\
Generated explanation & Abstractive & \emph{breakfast sandwiches are overall very filling}
\end{tabularx}
\caption{Examples of generated justifications for a recommendation of \emph{Shake Shack} to a particular user. From \cite{ni2019justifying}.
\label{fig:justification}
}
%\todo{table formatting}
%\todo{Could add other examples, show where dataset comes from}
\end{figure}

\subsection{Conversational Recommendation}
%\todo{Dunno if this goes here (probably not)}

Survey on conversational recommendation:
%https://arxiv.org/pdf/2004.00646.pdf
%\todo{write about, copy some refs from}

Arguably, our implicit goal when developing systems for text-based explanation or justification above is to mimic the ways humans explain or justify their decisions.
%Beyond our goal above
%\todo{only if actually above}
%of generating text-based explanations or justifications of recommendations, 
Following this, perhaps an ideal system 
%various attempts have been made to develop systems 
for interactive recommendations might mimic the paradigm of \emph{conversation}.

%\todo{More paragraph}

\paragraph{Query Refinement}

%A personalized system for conversational recommendations
Early approaches for conversational recommendations essentially use conversation as a form of 
iterative query refinement. In
\cite{thompson2004personalized}
users are asked questions that attempt to determine their preferences or constraints toward a fixed set of attributes (e.g.~cuisine type, price range, etc.); as such a user model is simply a weighting over potential attribute values. Retrieval then consists of selecting items whose attributes most closely match user preferences.
%https://arxiv.org/pdf/1107.0029.pdf

Other early approaches to `conversational' recommendation 
%treat conversation
are essentially forms of interactive recommendation, in which a system can query users or gather feedback from users during each round
%Improving recommender
%systems with adaptive conversational strategies
\cite{mahmood2007learning,mahmood2009improving}.
\cite{mahmood2009improving} adopts a reinforcement learning approach to interactive recommendation. During each step, the system may perform a number of actions, including asking users more detail about specific attributes, or asking them for feedback on a panel of recommendations. Reinforcement learning techniques are used to select an optimal policy in terms of what actions the system should perform under a given state.
%Early paper
%\todo{pretty useless description of this paper}

% Enhancing the diversity of conversational collaborative recommendations: a comparison \cite{kelly2006enhancing}
%Don't know if relevant

%Towards Conversational Recommender Systems 

\paragraph{Interactive Recommendation}

\cite{christakopoulou2016towards}
follows a similar strategy to those above, though embeds the interactive recommendation framework within a latent factor model, similar to those we studied in \Cref{chap:recsys}.
Here, the goal is to use a conversational strategy to quickly infer users' preferences $\gamma_u$ toward item properties $\gamma_i$.
Interactions consist of
a one-sided form of `converversation' in which the system asks 
simple questions to the user about their preferences;
%questions to the user, which are drawn from a set of templates.
%Templates
a main goal of \cite{christakopoulou2016towards}
is to understand the ideal characteristics of questions that should be asked to user in order to elicit their preferences efficiently.
Questions can be \emph{absolute} (ask a user whether they like or dislike an item), or \emph{pairwise} (ask a user which of two items they prefer). Question selection strategies consist of determining which items should be compared during each step.
Strategies include selecting random items, items with the highest estimated compatibility, or items whose compatibility has the most uncertainty.
After each question, compatibility scores are recomputed based on question responses, resulting in a model that gradually becomes more accurate during successive rounds.
%https://www.kdd.org/kdd2016/papers/files/rfp0063-christakopoulouA.pdf
%Is this the paper I'm thinking of that introduces the dataset? No.
%\todo{could say something about results, but not easy to summarize. best technique is thompson sampling}

%\todo{Table for this section?}

\paragraph{Free-form Conversation}

More recent approaches try to follow the conversational paradigm more literally. Ideally, conversations should be \emph{free-form}, in which both the system's question and the user's response take the form of free text.
%Toward Deep Conversational Recommendation 
\cite{li2018towards} attempted to formulate conversational recommendation in this form, for the specific task of movie recommendation.
%
A major challenge which they overcome is to build appropriate ground-truth data for this task.
Their approach builds on previous attempts to build dialog datasets, including dialogs specifically focused around movies \cite{dodge2015evaluating}.
%
Dialogs
%\todo{Why is dialog a word but dialogs isn't?}
are constructed by crowd workers, who assume roles of a \emph{recommender} or \emph{seeker}; conversations between the recommender and the seeker are tagged in terms of the movies mentioned, as well as explicit feedback (has the seeker seen the movies mentioned and did they like them). Around ten thousand such conversations are collected.

Having collected such data, \cite{li2018towards} then seek to train a dialog generation model that can fulfil the role of the recommender. Their solution combines ideas from dialog generation with a recommender system, so that users' preferences can be estimated and the output controlled to reference specific movies.
%\todo{rough}

\begin{figure}
\begin{tabularx}{\linewidth}{XXX}
Reference & Type & Description\\
\cite{thompson2004personalized} & Query refinement & Elicits users' preferences and constraints with regard to item attributes.\\
\cite{mahmood2009improving} & Reinforcement learning & Queries users about recommendations attributes during each round; learns a policy to choose queries to efficiently yield a desirable recommendation. \\
\cite{christakopoulou2016towards} & Iterative recommendation & Collects feedback about recommended items in order to iteratively learn user preferences; explores various query strategies to elicit preferences quickly. \\
\cite{li2018towards} & Free-form conversation & Collects conversational data in which a `recommender' suggests movies and a `seeker' provides feedback. Trains a dialog model to mimic the role of the recommender. \\
\cite{kang2019recommendation} & Free-form conversation and RL & Similar to the above, though trained using reinforcement learning so that the `recommender' and `seeker' exchange information to arrive at a target recommendation.
%\todo{Don't know if RL is ever defined}
\end{tabularx}
\caption{Summary of approaches for conversational recommendation
\label{fig:conversational}
}
%\todo{table formatting}
\end{figure}
%\todo{Put table in right place}

%\todo{Check all arXiv references in the bib for correct venues}


A potential limitation of \cite{li2018towards} is that it relies on explicit movie mentions and feedback to learn user preferences; as such their method could not straightforwardly recommend a movie based on a simple goal, such as a user requesting ``a good sci-fi movie.''
%
\cite{kang2019recommendation} seek to build conversational recommenders following this `goal oriented' view of recommendation.
To collect data, a conversation game is conducted, where both the seeker and recommender (`expert') are given prompt movie sets: the seeker's set represents their ostensible interests, while the expert's set is a collection of movies, one of which matches the seeker's preferences (determined offline based on a similarity metric). The expert's goal is to determine \emph{which} movie in their set matches the user's preferences via repeated conversation turns. \cite{kang2019recommendation} note that while in principle the expert could simply enumerate movies in their set until they reach the `correct' one, though in practice this rarely happens, and players tend to engage in free-form conversation, asking about general attributes and qualities.

Like \cite{li2018towards}, \cite{kang2019recommendation} then train dialog agents to play the game. During play, the expert can either engage in a dialog turn or select one of the movies from their set

%Recommendation as a Communication Game: Self-Supervised Bot-Play for Goal-oriented Dialogue
%\cite{kang2019recommendation}

\paragraph{Critiquing}
%\todo{Probably move up to explanation/justification/interpret section}
A Ranking Optimization Approach to Latent Linear Critiquing in Conversational Recommender System \cite{li2020ranking}
%Don't know if relevant


%\subsection{Personalized Dialog}
%\todo{Maybe}

\section{Case Study: Google's \emph{Smart Reply}}

Google's \emph{Smart Reply} is a system developed for \emph{GMail} to automatically recommend short responses for e-mail (\Cref{fig:smartreply}). 
%\todo{shrink figure, convert to black and white}
Given an e-mail thread as input, the system is tasked with surfacing (three) likely responses; although the goal is ultimately to maximize engagement with the feature, the system is trained by taking a large corpus of thread/response pairs, and learning to predict (or maximize the likelihood of) users' historical responses to a given thread.
%\todo{somewhat clunky}

\begin{SCfigure}
\fbox{\includegraphics[scale=0.125]{fig/smartreply.png}}
\caption{Examples of short responses suggested by Google's \emph{Smart Reply}. \label{fig:smartreply}}
\end{SCfigure}

As a case-study in personalized text generation, this system is interesting for several reasons:
\begin{itemize}
    \item As a form of personalized machine learning, Google's solution does not have explicit user parameters (i.e., it is similar to the non-parametric systems described in
    %\todo{where?}).
    Rather, `personalization' is done implicitly by learning from the context already present in the e-mail thread.
    \item Google describes two successive---and quite different---solutions to this problem in a sequence of papers. The first is based on a sequence-to-sequence language model (i.e.,~a text generation framework); the second is a seemingly more trivial nearest-neighbor based solution.
    \item Other interesting facets of the studies include how they deal with scalability, diversification, appropriateness of suggesting a reply for a given context, etc.
\end{itemize}

The models we will discuss are described in \cite{kannan2016smart} and \cite{henderson2017efficient}. We will mainly discuss the latter, more recent paper, as the solution mostly supercedes the former one, though we will make comparisons between both.



\section{Case Study: Personalized Recipes}
\label{sec:recipeCaseStudy}
%\todo{Could move this to same section as review generation section, and not call it a "case study"}

Given the significant variation in people's tastes and dietary needs, \emph{recipes} have recently emerged as an interesting source of textual data, both for personalized retrieval, and more recently, generation \cite{li2020recipes}.

Early systems to facilitate personalized interactions with recipes did so by helping users to search for recipes whose ingredients target specific health conditions \cite{ueta2011recipe}. Later systems approached the same task with explicit rules, and constraints of ingredients to avoid given specific dietary restrictions
\cite{inagawa2013support}. Other retrieval-oriented systems have sought to help users find recipes `in the wild,' e.g.~by searching for recipes that correspond to a photo \cite{marin2019recipe1m}.

More recently, \cite{majumder2019generating} considered whether personalized recipes can be synthesized using text generation frameworks. Here, the idea is to generate novel recipes that are consistent with a particular user's preferences (or specifically, their previous interactions). The setup is somewhat similar to that of \Cref{sec:reviewGeneration}, where we sought to generate personalized text conditioned on a (representation of a) user and an item. In \cite{majumder2019generating}, the training objective consists of a set of recipes a particular user has consumed, based on which the system should output (i.e.,~generate the text of) another recipe the user would consume.\footnote{In practice, \cite{majumder2019generating} inputs other metadata (such as a recipe title) to the method to overcome the difficulty of generating recipes `from scratch.'} In this way, the method can generate recipes that are consistent (in terms of e.g.~ingredients, cooking techniques, etc.)~with those that the user would normally consume. An example of a recipe generated with this system is included in \Cref{fig:recipeGenerated}.

\cite{shuyang?}
%\todo{only cite follow-up paper if it is accepted}


\begin{SCfigure}
\parbox{0.5\textwidth}{
\textbf{Name:} Pomberrytini; \textbf{Ingredients: pomegranate-blueberry juice, cranberry juice, vodka}
%; Calorie: Low

Combine all ingredients except for the ice in a blender or food processor. Process to make a smooth
paste and then add the remaining vodka and blend until smooth. Pour into a chilled glass and
garnish with a little lemon and fresh mint.
}
\caption{Example of a personalized recipe, from \cite{majumder2019generating}. \label{fig:recipeGenerated}}
\end{SCfigure}

\begin{figure}
\parbox{0.48\textwidth}{
\textbf{Chicken Dijon}

\textbf{Ingredients:} Butter, Chicken, Chicken broth, Dijon mustard, Flour,Light cream, Onion, Pepper,Salt, Wine

1) Sprinkle the chicken breasts with onion powder, lemon pepper and salt. 2) Saute the chicken in the butter for 20 min, or cooked through and tender. 3) Remove chicken to a platter and keep warm. 4) Measure the pan juices and enough chicken broth to make 1 cup liquid. 5) Return the stock mixture to the pan and add the wine. 6) Stir together the light cream and the flour till smooth. 7) Add to the broth, cook and stir till smooth, thickened and bubbly. 8) Stir in the dijon mustard, stir till smooth. 9) Return the chicken to the pan, or serve the sauce separately.

}\hspace{0.04\textwidth}\parbox{0.48\textwidth}{
\textbf{Low-Calorie Chicken Dijon}

\textbf{Ingredients:} Chicken, Dijon mustard, Flour, Pepper, Salt, Carrot, Garlic, Mushroom, Soy sauce

1) Preheat oven to 350 degrees f. 2) Place chicken breasts, breast side up, in a shallow roasting pan. 3) Combine soy sauce, mustard,garlic and salt. 4) Brush over both sides of the chicken breasts. 5) Bake for 40 minutes. 6) Meanwhile, heat a small sauce-pan over medium heat. 7) Stir in carrots, mushrooms and pepper. 8) Cook and stir for 5 minutes. 9) Stir in flour. 10) Pour over chicken breasts, and bake for an additional 10 minutes until chicken is cooked through. 11) Serve chicken with carrots and sauce.
}
\caption{Example of a recipe (left) and a modified version (right) targeting a specific dietary goal \cite{?}. \label{fig:recipes}}
\end{figure}
%\todo{Ugly formatting in figure}


\section{Exercises}

The exercises below may be conducted using any dataset that includes reviews associated with numerical ratings.
%\todo{say more, also formatting is ugly}

\begin{exercises}
\item Implement a sentiment analysis pipeline based on a bag-of-words model, as in \Cref{sec:bagOfWords}, \Cref{eq:sentiment1}. You may follow the code provided as a starting point to build a model based on the 1,000 most common unigrams.
%; this exercise may be conducted using any dataset that includes reviews associated with numerical ratings.
%\todo{moved above}
\label{ex:text1}
\item Several choices must be made when building feature representations from text. Extending your model from \Cref{ex:text1}, and using a validation set, experiment with various modeling choices (in addition to the regularization hyperparameter $\lambda$). Possible modeling choices include:
\begin{itemize}
    \item The dictionary size;
    \item Whether to use unigrams, bigrams, or a combination of both.\footnote{That is, all unigrams \emph{and} bigrams can be sorted by popularity;
    %\todo{finish}
    }
    \item Whether to remove capitalization, punctuation, or to treat punctuation characters as separate words;
    \item whether to use tf-idf
    %\todo{style of tf-idf}
    %\todo{finish}
\end{itemize}
%we should be careful to tune the model to avoid overfitting. Split the data into train, validation, and test sets, and use the validation set to find a regularization hyperparameter $\lambda$ as well as a dictionary size that works better than the solution you found above.
\item Something with item2vec
\item 
\label{item:qTFIDF}
%Using TF-IDF for item-to-item similarity (could be project)
As discussed in \Cref{sec:?}, tf-idf representations of documents
%\todo{tf-idf or TF-IDF (throughout)?}
%\todo{Is this actually discussed anywhere? should be}
can be used to retrieve documents that are semantically similar to each other. For this exercise, we'll use all reviews of a single item as a single `document.' First, compute the tf-idf representations of all items, and compute the most similar item (in terms of cosine similarity between tf-idf vectors) given a particular query (e.g.~the first item in the dataset). Compare this to similarity computed on bag-of-words representations.
\item In \Cref{chap?,exercise?}, we considered using similarity functions to predict ratings for user-item pairs. Adapt one of those predictors (e.g.~the predictor from \Cref{eq:simPredict?}) to estimate ratings using the item-to-item similarity function you developed in \Cref{item:qTFIDF}.\footnote{Given that repeatedly computing item-to-item similarities for high-dimensional text features is likely quite computationally intensive, it is most likely feasible to evaluate your method only on a small sample of user-item pairs.}
\end{exercises}

\stepcounter{projectCounter}
\subsection{Project \theprojectCounter: Personalized Document Retrieval}
%\todo{Rename project}

In the above exercises, 
%\todo{check that the above is true}
we have focused on building predictive models using features derived from text. In fact, most of the models we developed were not \emph{personalized}. Here, we'll explore how to personalize these models following the approaches we developed in this chapter.

%\todo{say what dataset we're using!}

As a starting point, we'll build a personalized model of sentiment analysis; your model can extend the one you developed in Exercises \Cref{7.1and7.2}.
Perhaps the simplest form of personalization consists of fitting bias terms for each user, much like the bias terms we included when developing recommender systems in \Cref{chap:recommender}. Such a term can account for the fact that one user may regard (e.g.)~a three-star rating as positive (and therefore use positive language)
%\todo{therefor or therefore?}
whereas anther user may regard it as negative. This term can be incorporated by extending a model like that in \Cref{eq:?}:
\begin{equation}
    \alpha + \beta_u + \sum_{w \in \mathcal D} \text{count}(w) \cdot \theta_w.
    \label{eq:projBoW1}
\end{equation}
This model could be fit either by (a) treating the user identity as a one-hot vector, and treating the problem as an ordinary linear regression problem, or (b) by gradient descent, much as we fit bias terms when developing recommender systems in \Cref{sec:?}.

Having fit this model, investigate the extent to which the addition of the bias term improves performance (e.g.~in terms of the MSE), and the extent to which it alters the weights associated with the most positive and negative words.

Following this, develop a more complex personalized model that estimates a user's compatibility with a document in terms of latent factors. Rather than associating latent factors with individual documents, we'll associate them with \emph{words} in documents, augmenting our bag-of-words model from \Cref{eq:projBoW1}
\begin{equation}
    \alpha + \beta_u + \sum_{k = 1}^K \sum_{w \in \mathcal D} \text{count}(w) \cdot \gamma_{u,k} \theta_{k,w}
\end{equation}
%\todo{finish}



\chapter{Personalized Models of Visual Data}
\label{chap:visual}

Many of our decisions are guided by visual factors, and preferences toward visual attributes can be highly subjective.
%\todo{not connected to rest of paragraph, maybe move below}
Traditional models of visual data deal with problems like classification, detection, or more recently image generation, though the bulk of approaches are not personalized:
discriminative models (classifiers, detectors) are usually concerned with identifying some objective label in an image,
or generative models are concerned with learning a background distribution governing the overall dynamics of a large corpus of data.

The situation above is much as we saw when introducing models for text in \Cref{chap:text}. Likewise, just as we saw in \Cref{chap:text}, many problems involving visual data depend on subjective factors or user context.
%\todo{not connected}
Just as we saw in \Cref{sec:recText} that text can be used to improve both the fidelity and interpretability of recommendations, visual data can likewise be included to improve the accuracy of models in settings where personal preferences are significantly guided by visual signals.

Visual data are critical in domains like fashion, 
where preferences are largely guided by visual factors.
%Furthermore
%. For one, fashion recommendation problems are 
Problems like recommendation in such settings are
highly personalized, and problems like compatibility among items depend on complex factors that are hard to precisely define. 
%Presumably, visual attributes play a key role in users' decisions. 
Furthermore, recommendation in scenarios frequently suffers from \emph{cold-start} (or `cool start') problems, given the long tail of new and rarely consumed items.

We'll begin this chapter by exploring how to incorporate visual data into recommendation approaches.
Much of our discussion will be centered around domains like fashion recommendation, where visual features naturally play a key role, though we'll also look at other visually-guided scenarios ranging from art to home decor.

Following this we'll explore new recommendation modalities involving visual data. Item-to-item, or set-based recommendation are particularly important in settings involving visual data, again including settings like outfit generation in fashion.
%Like the text-based models 
%\todo{No section refs above?}

Finally we'll explore personalized \emph{generative} models of images. Just as we saw models that generate personalized text in \Cref{chap:text}, there are a few settings where one might wish to generate images that are personalized to a user's preference or context, such as systems for personalized design.

\section{Visually-Aware Recommendation and Personalized Ranking}
\label{sec:visualRec}


%Both of the above issues suggest 
The issues we introduced above suggest the possibility of 
systems for recommendation and ranking that make use of visual features. Much as we saw with text in \Cref{sec:?}, visual data is difficult to incorporate directly into recommender systems, given that feature representations are high-dimensional, and dense. Below we discuss a few of the main approaches to incorporate visual data into recommendation and personalized ranking models.

\subsection{Visual Bayesian Personalized Ranking}

Initial attempts to incorporate visual data into ranking models extend the Bayesian Personalized Ranking framework from \Cref{sec:BPR} to incorporate observed image features $f_i$ associated with each item, such as a product image from an e-commerce website.

That is, we want to define a compatibility function $x_{u,i,j} = x_{u,i} - x_{u,j}$ (as in \Cref{eq:BPRCompat}) that estimates which of two items $i$ and $j$ are more compatible with the user.

Starting with a simple latent-factor based compatibility model, and given our goal of building a system that works in (item) cold-start scenarios, we might first consider simply replacing our (latent) item representations $\gamma_i$ with our observed image features, i.e.,
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + \gamma_u \cdot f_i.
    \label{eq:vbprAttempt1}
\end{equation}
In this way, $\gamma_u$ would now determine which features are most compatible with each user (in fact, this is a linear model as in \Cref{chap:basics}).
Although conceptually reasonable, the issue in doing so becomes quickly apparent once we consider that image features are typically very high dimensional. For instance, visual features extracted from ImageNet (as used in \cite{he2015vbpr}) are 4,096 dimensional. Incorporating them directly into $x_{u,i}$ as in \Cref{eq:vbprAttempt1} would thus require fitting thousands of parameters \emph{per user}, which is not feasible in datasets that typically consist of only (e.g.)~tens of interactions per user.

\emph{Visual Bayesian Personalized Ranking} \cite{he2015vbpr} attempts to address this by projecting images into a low-dimensional embedding space via a matrix $\mathbf{E}$. Here $\mathbf{E}$ is $|f_i| \times K$ matrix (e.g.~$4,096 \times K$), 
%\todo{Don't think 4,096 renders right in math mode}
which projects the image into a $K$ dimensional space. Following this, the projected image dimensions can be matched to user preference dimensions:
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + \gamma_u \cdot (\mathbf{E}f_i).
    \label{eq:vbprAttempt2}
\end{equation}
Note that the projected features $\mathbf{E}f_i$ fulfil much the same role as $\gamma_i$ in a typical latent factor model, except that they are learned based on observed features. 

Note that $\mathbf{E}f_i$ is a \emph{learned} embedding, and is fit so as to maximize the probability of observed interactions, as with all other terms in \Cref{eq:vbprAttempt2}. Note also that while $\mathbf{E}$ is high dimensional (e.g.~around 40,000 parameters if $|f_i|=4,096$ and $K=10$), it is a \emph{global} term that is shared among all items; thus for a large enough dataset it accounts for only a small fraction of the model's parameters.

Because the embedding is low-rank, we are assuming that users' preferences toward these visual dimensions can be explained via a small number of factors. While this is similar to the assumption made by a `standard' latent factor model (e.g.~as in \Cref{eq:latentFactor?}), we are further assuming that these factors can be explained by visual dimensions. However in practice there could be several latent factors \emph{not} explainable by visual features (e.g.~factors due to price, material, brand, etc.) To address this the original paper includes \emph{both} latent item factors $\gamma_i$ as well as visual item factors $\mathbf{E}f_i$:
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + 
    \underbrace{\gamma_u(\mathbf{E}f_i)}_{\text{visual preference dimensions}} +
    \underbrace{\gamma'_u \cdot \gamma_i}_{\text{latent preference dimensions}} + \underbrace{\beta^{(f)} \cdot f_i}_{\text{visual bias}}.
    \label{eq:vbpr}
\end{equation}
Correspondingly, there are two sets of user terms: $\gamma_u$, which explains preferences toward visual factors, and $\gamma'_u$, which explains preferences toward non-visual factors. Intuitively, the two terms will play different roles depending on how `cold' an item is: for a cold (or `cool') item, visual features will be much more reliable than latent factors; whereas for 
`hot' items (i.e.,~those with many associated interactions)
%\todo{Have I already defined these terms?}
$\gamma_i$ will be able to capture additional non-visual dimensions. \Cref{eq:vbpr} also includes a `visual bias' term $\beta^{(f)}\cdot f_i$ ($\beta^{(f)}$ is a $|f_i|$-dimensional vector) that is able to estimate item biases in cold scenarios.

Several other considerations must be made to implement such an algorithm efficiently. For instance, accessing the (high-dimensional) image features at random (e.g.~within a stochastic gradient ascent algorithm), leads to poor caching performance; likewise computing the projection $\mathbf{E}f_i$ is expensive. In practice these issues are dealt with by pre-computing all projections $\mathbf{E}f_i$ (which can be performed as a single matrix-matrix product), and updating $\mathbf{E}$ only periodically during gradient ascent.

Visual Bayesian Personalized Ranking is effective in settings where items have few associated interactions. In the original paper it is demonstrated on a clothing dataset from \emph{Amazon}, as well as a clothing trading dataset (\emph{tradesy}).
%\todo{Check spelling of tradesy, consistent italics}
The latter is particularly challenging because traded items are not associated with long transaction histories, meaning that model predictions must largely rely on visual signals.

\subsection{Case Study: Modeling the Visual Evolution of Fashion Trends}
%\todo{Probably demote this to a paragraph}

\cite{he2016ups} extended the above ideas from Visual Bayesian Personalized Ranking to incorporate temporal dynamics. Modeling temporal dynamics in this setting is interesting partly because the patterns of temporal variation in (e.g.)~clothing purchases are different from those that were successful in other settings, such as on \emph{Netflix} (\Cref{sec:caseStudyNetflixTemporal}).

The main idea in \cite{he2016ups} is simply to break the training dataset into a sequence of \emph{epochs}, each of which have their own parameters. These epochs are somewhat akin to the `bins' used to model long-term temporal dynamics on Netflix, though a key difference is that the bin sizes are variable and placed at learned intervals (using a dynamic programming procedure); given that the model has a large number of parameters, this helps to ensure that fewer bins are used during time periods with little temporal variation, whereas more (and smaller) bins are used during periods which are more dynamic.

Potentially, models of temporal dynamics in fashion are also interesting because they might reveal historical trends in fashion over time.

% Mention point about projecting images into low dimensional space only once per iteration to save on computational costs

\section{Visual Compatibility of Items}

%Look for refs in WC Pinterest paper

In the previous section
%\todo{Is it the previous section?}
we saw how visually-aware recommender systems can be used to match items (or images of items) to users' preferences. In domains like fashion, there is also interest in establishing compatibility between items (or recommending one item on the basis of another), for example to generate outfits, or even wardrobes of mutually compatible items. In \Cref{sec:neighborhood} we discussed several types of recommendation approaches that considered similarity between items; such measures guide `item-to-item' recommendation approaches, as in \Cref{fig:alsoviewed}.

Here, we would like to develop similar approaches that establish visual similarity (or compatibility) between items. Rather than basing similarity on interaction histories as we did in \Cref{sec:neighborhood}, here we can base similarity directly on the visual appearance of items.

\section{Case Studies: Fashion Compatibility}
\label{sec:fashionCaseStudies}
%\todo{Decide whether there should be sections with multiple case-studies like this}

Many studies on visual compatibility are specifically concerned with \emph{fashion images}. Estimating compatibility in such a domain has obvious applications to specific tasks like outfit generation and recommendation. More broadly, in settings like fashion, 
%interactions tend to be guided by compatibility relationships.
compatibility with past interactions or purchases is a strong predictor of future interactions.

Some of the specific characteristics that makes this problem difficult (and different from other forms of (e.g.)~item-to-item recommendation, are as follows:
\begin{itemize}
    \item It is challenging to construct datasets that act as `groundtruth' for visual compatibility, i.e.,~pairs of items that are known to `go well' together.
    \item Further to the above, any groundtruth of compatible items are bound to be highly noisy, and highly subjective; successful methods need to account for these challenges, and possibly learn compatibility in a personalized way.
    \item The features that make items visually compatible in settings like fashion could be incredible subtle, and could be quite different from the information available in co-purchase data, or even in most visual feature descriptors.
    \item Finally, the notion of `compatibility' is semantically quite different from `similarity.' E.g.~clothing items that go together should be similar in some ways but complementary in others.
\end{itemize}

Approaches to these problems mainly differ in their specific solutions to the problems above. We describe a few key papers below.

\subsection{Estimating Compatibility from Co-purchases}
\label{sec:hengelPaper}

Early approaches to estimating visual compatibility built datasets from co-purchases, for example using publicly-available datasets of reviews from \emph{Amazon}.
%\todo{how to italicize company names etc.}
%\todo{Is amazon data already mentioned above? Only mention it once.}

\cite{mcauley2015image} crawled data from Amazon's surfaced recommendations (`people who bought X also bought Y' etc.~as in \Cref{fig:alsoviewed}), and, in the case of clothing, treated these as `groundtruth' examples of items that are visually compatible.

Having defined such a compatibility function, the goal is to learn an appropriate distance function, such that frequently co-purchased items tend to be closer together than others. The distance function is then used in a simple binary classification framework (similar to logistic regression) to predict:
\begin{equation}
    p(i \text{\ co-purchased with\ } j) = \sigma(c - d(i, j)).
\end{equation}
Previously, we considered how to learn distance functions for problems such as next Point-of-Interest recommendation (\Cref{sec:distancePOI}). When doing so,
%\todo{Check whether above is true of all cases}
items (and users) were projected into a latent space via parameters $\gamma$. To recommend compatible clothing, we might instead use features extracted directly from (the product images of) $i$ and $j$: First, general-purpose visual features are readily available, and are likely to be informative in fashion compatibility scenarios; second, reliance on features is desirable in cold-start settings, which might be common in settings (like fashion) where item vocabularies are large and changing; third, a model based only on visual features can be more straightforwardly \emph{transferred} to settings where user data is not available.

Given image features associated with the items $f_i$ and $f_j$, \cite{mcauley2015image} discusses several strategies for establishing visual similarity. Trivially, one could directly consider the (squared) distance between $f_i$ and $f_j$ (i.e.,~$\| f_i - f_j \|_2^2$), however general-purpose image features may not focus on attributes that are relevant to fashion.

A second solution is to learn a weighted distance function that discovers which features are relevant and discards those that are not, i.e.,~$\sum_k w_k (f_{i,k} - f_{j,k})^2$. However it is argued that in fashion scenarios `compatibility' cannot be captured by modeling similarity between features---for example, a user generally would not select a shirt because it looks `similar' to a pair of pants. To address this, a similarity function is proposed which projects the images into a low-dimensional `style space':
\begin{equation}
    d(i,j) = \| s_i - s_j \|_2^2; \quad \text{where} \quad s_i = M \times f_i.
    \label{eq:hengelPaper1}
\end{equation}
In the case of \cite{mcauley2015image} $f_i$ is a 4096-dimensional image descriptor, extracted from a model trained on ImageNet \cite{jia2014caffe}; $M$ is then a $4096 \times K$ vector, where $K$ is some small embedding dimension (on the order of $K=10$). Ultimately the embedded vector $s_i = M\times f_i$ is analogous to the latent vectors $\gamma_i$ from previous models, in the sense that it captures the underlying dimensions that explain variation in co-purchases.

The method is then trained using a dataset $\mathcal C$ of complementary pairs, along with a set of \emph{non-complementary} pairs $\mathcal C^-$ (which in practice are sampled randomly). The model is then trained using a logistic regression-like setup to distinguish complementary and non-complementary pairs:
%\todo{Above seemed missing, so added, but check to see if it fits in the organization, and that it's not the same as the text used in Sceptre description}
\begin{equation}
\underbrace{\sum_{(i,j) \in \mathcal C} \log \sigma(c - d(i, j))}_{\text{complementary pairs}} + \overbrace{\sum_{(i,j) \in \mathcal C^-} \log (1 - \sigma(c - d(i, j)))}^{\text{non-complementary pairs}}
\label{eq:hengelPaper2}
%\todo{equation numbering?}
\end{equation}

Finally, although the approach is mainly designed for item-to-item recommendation (and as such is not \emph{personalized}), a personalized version can be developed by adding a user latent vector $\gamma_u$ that encodes which dimensions of this `style space' are important to them:
\begin{equation}
    d_u(i,j) = \sum_k (\gamma_{u,k} s_{ik} - \gamma_{u,k} s_{jk})^2
    %\todo{fix (u,k) vs (uk) everywhere
\end{equation}
(in this case, the model is trained on triples $(u,i,j)$ of co-purchases of items by each user $u$).

%\todo: Note similarity compared to cold-start paper from previous section (both bilinear models)
Pairwise Preference Regression for Cold-start
Recommendation

Ultimately, the paper shows that this type of model can be used in several ways. First, it can predict co-purchases accurately, especially when predictions are personalized. Second, the use of image data is effective at \emph{visualizing} the parameters of the model, i.e.,~determining what are the primary dimensions that explain variance in users' `styles.' Finally, since the model (as in \Cref{eq:hengelPaper1}) takes only images as input, it can be transfered to assess compatibility (and arguably, `fashionability') of outfits outside of the original training data.

%Various other papers have made use of the same co-purchase data, or followed up on the basic idea above. For example, 
\cite{veit2015learning} made use of the same co-purchase data to solve the same task, but did so directly from the `pixel level,' i.e.,~by training a Convolutional Neural Network, rather than using a pre-trained image representation. This type of architecture is depicted in \Cref{fig:Siamese}: two input images (items), labeled as `compatible' or `incompatible,' are passed through two CNNs, both of which share the same parameters. The CNNs learn low-dimensional representations $\phi(x)$ and $\phi(y)$ for the two items; these are essentially equivalent to the `style space' embeddings of \Cref{eq:hengelPaper1}, except that they are learned from the pixel level, and therefore could potentially capture subtle characteristics not available in pre-trained representations. Like \Cref{eq:hengelPaper1}, the model is trained to learn a metric, so that compatible items have nearby embeddings and incompatible items do not. 

\begin{figure}
%tikz code from https://github.com/PetarV-/TikZ/blob/master/Generative%20adversarial%20network/README.md
\centering
\begin{tikzpicture}
[
ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        thick,
        },
]

	\node (z) {item $y$};
	\node[circle, draw, thick, right=5em of z] (x) {$\phi(y)$};
	\draw[-stealth, thick] (z) -- node[above] {CNN} (x);
	%\node[left=of z] (i) {};
	%\draw[-stealth, thick] (i) -- node[above] {$p_\theta(\vec{z})$} (z);
	\node[above=of x, circle, draw, thick] (xt) {$\phi(x)$};
	\node[left=5em of xt] (it) {item $x$};
	\draw[-stealth, thick] (it) -- node[above] {CNN} (xt);
	\node[thick, right=4em of x, yshift=3.1em, xshift=-0.9em] (D) {$\| \phi(x) - \phi(y) \|$};
	\node[right=2em of D] (out) {compatible?};
	\draw[-stealth, thick] (D) -- (out);
			
% 	\node[right=2.5em of x, circle, fill, inner sep=0.15em] (pt1) {};
% 	\node[right=2.5em of xt, circle, fill, inner sep=0.15em] (pt2) {};

			
	\draw[dashed, thick] (x) edge[bend right] (xt);
			
	\node[circle, draw, thick, fill=white, inner sep=0.15em, left=of D] (pt3) {};
			
% 	\draw[-stealth, thick] (x) -- (pt1);
% 	\draw[-stealth, thick] (xt) -- (pt2);
	\draw[-stealth, thick] (pt3) -- (D);

\end{tikzpicture}
\caption{Basic Siamese setup for item-to-item compatibility. \label{fig:Siamese}}
%\todo{edit this for this specific case, improve caption}
%\todo{also add \gamma_u to the input to the generator}
\end{figure}

\cite{he2016learning} showed that prediction performance can be improved in fashion settings by using separate embeddings for the `query' and `target' items $i$ and $j$, i.e.,
\begin{equation}
    d(i,j) = \| \gamma_i - \gamma'_j \|_2^2.
\end{equation}
Critically, by using two different latent spaces, `compatibility' need not follow the assumptions of a metric space, e.g.~an item need not be compatible with itself. In this way the model can learn which aspects should be systematically different when matching items---such as blue pants going with brown shoes.
%\todo{A little too much about my own papers, could shrink}

\subsection{Learning Compatibility from Images in the Wild}
%\done{I don't think there's anything else to add?}

The above papers showed that visual data can be effective when predicting co-purchases in domains like fashion. However, it is arguable whether such models actually learn a useful notion of `fashionability.' For one, a co-purchase is a very noisy indicator of whether two items are compatible: in practice, two purchases on one account may not even be for the same person. Secondly, the images in such datasets (e.g.~product images from Amazon) are not `wild' images (they are usually scaled, centered objects on a white background), so could not be used to assess the fashionability of an outfit in a photo (for example).

\cite{kang2019complete} tried to address this issue by developing models of fashionability that operated directly on images in the wild. At training time, the approach consists of an image (or `scene') that is known to contain a particular item; each item also has a `clean' product image much like those in \Cref{sec:hengelPaper}. The basic idea is then that item must be compatible with other object in the scene.

%\todo{mention street2shop dataset (from WC paper), maybe in next para}

To build a training set, the known items are cropped from each training image, each resulting in a scene image that does \emph{not} contain the known item, but ought to contain compatible objects. Ultimately, this means that we separately have an item, plus a (cropped) scene that is known to be compatible with the object, but does not contain it. From here, fashion compatibility can be estimated by learning a relationship between the scene and product images:
\begin{equation}
    d(s,p) = \| f(s), f(p) \|_2^2.
\end{equation}
Much like \Cref{eq:hengelPaper1}, this distance function is based on learned embeddings of the scene and product images ($s$ and $p$). Several differences exist between these embeddings and those used in \Cref{sec:hengelPaper}, mostly to account for the fact that the scene image likely contains a large number of \emph{irrelevant} objects. Critically, the method uses an \emph{attention mechanism} (see e.g.~\cite{xu2015show}), which is used to identify regions of the scene image that are relevant for compatibility detection. For example, in an outdoor image, the attention mechanism may learn to focus on the person in the image and their outfit, while ignoring `background' objects in the surrounding scene.

Note that the above is ultimately not a \emph{personalized} model, i.e.,~there are no parameters associated with the individual users. However the system is arguably more useful than that of \Cref{sec:hengelPaper} from a personalization perspective, in the sense that it would allow a user to upload an image of themselves and receive recommendations that complement their personal style. As such, it is an example of the type of \emph{contextual} or non-parametric personalization that we discussed in \Cref{sec:nonparametric}.

In addition to experiments on personalized fashion, \cite{kang2019complete} shows that the same technique can be used for complementary item recommendation in other settings, such as recommending compatible furniture based on items in a living room.

%\done{also mention the use for furniture}

%\todo{Mention that other ruining paper that learned non-metric embeddings?}

%\todo{Throughout book: see if I should use \bm (boldmath) for math in headings}

\subsection{Generating Fashionable Wardrobes}

The papers above consider \emph{pairwise} compatibility among items as a proxy for selecting sets of items that will form compatible outfits.

\cite{hsiao2018creating} considers the more challenging problem of finding \emph{collections} of items that can be used to generate compatible outfits (called `capsule wardrobes'). They note a general limitation of of much of the above work in terms of the difficulty and noise involved in collecting pairwise training data

Their specific notion of a `wardrobe' is a set of items belonging to fixed sets of categories (or `layers'), e.g.~tops, bottoms, outerwear. The suggested quality of good a wardrobe is that it should be capable of generating many outfits, i.e.,~a large number of sets of items should be mutually compatible. At the same time, a wardrobe should have a wide \emph{variety} of items (e.g.~many nearly-identical pairs of jeans and t-shirts would generate many outfits but would not be a good wardrobe).

%The approach proposed by \cite{hsiao2018creating} to defining the quality of a single outfi
First \cite{hsiao2018creating} defines a measure to determine the compatibility of items in a single outfit. Their specific approach is based on \emph{topic modeling} which we studied a little in \Cref{sec:reviewAware}. Essentially, an outfit is represented by a low-dimensional vector; each dimension of this vector in turn corresponds to a mixture over certain visual attributes (e.g.~an outfit dimension might correspond to `floral patterns,' and this dimension might be associated with visual attributes describing appearance, shape, cut, etc.).\footnote{`Outfits' and `attributes' are analogous to `documents' and `words' in the original topic model formulation \cite{blei2003latent}.}
%
A set of items is said to constitute a good outfit if their collective attributes resemble those of training outfits.

\cite{hsiao2018creating} explores the relationship between these interacting components (outfits, wardrobes, versatility, as well as personalization of these components), and develops optimization schemes to circumvent the combinatorial nature of the problem. 

Perhaps most important contributions in \cite{hsiao2018creating} are simply the creative use of training data, and the formalization of what it means for a set of items to be `good' in terms of compatibility.
%
\cite{hsiao2018creating} make the argument that this type of model and training procedure is preferable to other techniques that rely on e.g.~pairwise compatibility relationships; a topic model-based approach results in a holistic notion about the overall qualities of an outfit, and allows for training on complete outfit images rather (e.g.)~collecting training data from co-purchases (as in \Cref{sec:?}), which may be subject to noise or otherwise not representative of real fashion compatibility.

\subsection{Domains other than Fashion}

The visual dynamics of fashion items are often the focus of studies on personalized visual models, though fashion is not the only domain in which visual features play a key role.

A few others have sought to study personalized visual dynamics in related domains.
\cite{bell2015learning} learned models of visual compatibility of home decor items collected from \emph{Houzz};
%\todo{style}
the model is similar to the fashion compatibility model we studied in \Cref{sec:?}, with the main goal being to learn a similarity function between images via a Siamese network. 
%Here, rather than estimating whether two items of clothing would appear in an outfit, the model is trained to estimate whether two home decor items would appear together in a scene; a main challenge in this work is to build ground-truth data of visually compatible items in a scene
At training time, the problem is essentially cast as a form of visual search: that is, given a scene containing an item, identify the item. This is achieved by training on a dataset of image pairs, where one image consists of an item in a scene while the other consists of a clean (or `iconic') product image. As such, the learned distance metric simply attempts to map both \emph{in-situ} images and clean images to the same point. Although the main application for this task is visual search (i.e.,~find the item in this image, or find images containing this item), other potential applications are discussed that make use of the learned metric, such as identifying stylistically compatible items across categories.

%For instance the model of 
\cite{kang2019complete} also adapted their model (which we studied in \Cref{sec:?}) to the problem of furniture / home decor recommendation.
%\todo{make sure reference points to an earlier section}
%was also used to study \emph{furniture} recommendation (using a dateset of interior design and home decor items from \emph{Pinterest});
The technique is essentially the same as that described in \Cref{sec:?}, but is trained on a dataset of interior design and home decor items from \emph{Pinterest}.
%\todo{style}
Here, rather than estimating a clothing item which completes an outfit (by withholding that item from the scene during training), the method is used to 
%estimate a piece of furniture 
identify home decor items which are
%which is 
visually compatible (or complementary to) others in the scene.

%\todo{both paragraphs above are essentially walls of text}



%Learning visual similarity for product design with convolutional neural networks
%\cite{kang2019complete} (furniture)

%Kang paper (art) - also mention that there is a social aspect to art recommendation.
\cite{he2016vista} considered visual dynamics in the context of \emph{art} recommendation. Their setting is an online art community (\emph{Behance}), where personalized preferences can potentially be guided by a variety of visual, temporal, and social factors. Spiritually, the modelling approach is similar to that used to model the temporal dynamics of fashion (as in \Cref{sec:?}), i.e.,~by combining pre-trained image embeddings with a variety of application-specific temporal dynamics. The main component of their temporal model is a Markov chain-based approach (as in \Cref{sec:?}), whereby interactions with are largely guided by recent context. \cite{he2016vista} also observe that art recommendation has a significant social component, whereby preferences can be estimated based on the identity of the artist as much as from the art itself.

\subsection{Other Techniques for Substitutable and Complementary Product Recommendation}

While much of the work on substitutable and complementary item recommendation is motivated by applications in fashion (where such recommendations can naturally be used to generate outfits, etc.),
%\todo{not good sentence}
a few others have considered the problem more broadly, either to consider different modalities of data (besides visual features), or to consider complementarity in settings other than fashion.

\paragraph{Substitutes and Complements from Text}

\cite{mcauley2015inferring} attempted to estimate substitutable and complementary products using the same dataset of \emph{Amazon} co-purchase and co-browsing data as in \Cref{sec:imageSubstitutes}

\begin{equation}
\underbrace{\sum_{(i,j) \in \mathcal C} \log \sigma(\gamma_i \cdot \gamma_j)}_{\text{complementary pairs}} + \overbrace{\sum_{(i,j) \in \mathcal C^-} \log \sigma(\gamma_i \cdot \gamma_j)}^{\text{non-complementary pairs}} +
\lambda \underbrace{\sum_{(u,i) \in \mathcal T} \sum_{w \in d_{u,i}} \log p(w | \gamma_u, \psi)}_{\text{topic likelihood}},
\label{eq:sceptre}
\end{equation}

%Inferring Networks of Substitutable
%and Complementary Products
%https://cseweb.ucsd.edu/~jmcauley/pdfs/kdd15.pdf

\paragraph{Learning Asymmetric Product Relationships}
%\todo{Is "asymmetric" the right word? Really it's about identity, not symmetry. Think about it. Here and elsewhere.}

\cite{wang2018path} also studied the problem of recommending substitutable and complementary products, proposing several modifications to the approach used in \cite{mcauley2015inferring}.

First, as we saw in \Cref{sec:?}, a good model for complementary product recommendation ought to recognize that `complementarity' should ideally not be captured by a similarity function. That is, complementary items have systematically \emph{different} characteristics, and in particular an item is not complementary with itself.

Following this logic, a simple modification of the complementarity function in \Cref{eq:sceptre} consists of using two sets of factors $\gamma_i$ and $\gamma'_j$ for complementary pairs $i$ and $j$:\footnote{Note that in practice, complementary pairs $i \rightarrow j$ are \emph{directed}, e.g.~a large fraction of camera purchases are paired with a memory card, whereas only a small fraction of memory card purchases are paired with a camera.}
\begin{equation}
\underbrace{\sum_{(i,j) \in \mathcal C} \log \sigma(\gamma_i \cdot \gamma'_j)}_{\text{complementary pairs}} + \overbrace{\sum_{(i,j) \in \mathcal C^-} \log \sigma(\gamma_i \cdot \gamma'_j)}^{\text{non-complementary pairs}}
\end{equation}

Following this modification, several other extensions are proposed, mostly to handle data sparsity issues and cold-start scenarios. Whereas \cite{mcauley2015inferring} did so using signals from text, \cite{wang2018path} leverages knowledge about the specific semantics of substitutable items; for example, if complementary pairs tend to belong to specific sub-categories (e.g.~shirts are often compatible with jeans), this acts as a weak signal that other products within these categories are also complementary. Likewise, substitutable relationships might be \emph{transitive} (for example), i.e.,~if $i$ is substitutable for $j$, and $j$ is substitutable for $k$, then $i$ is likely substitutable for $k$; various `soft' constraints of these types ultimately help to improve performance of the model.

% A Path-constrained Framework for Discriminating
% Substitutable and Complementary Products in E-commerce
%https://dl.acm.org/doi/pdf/10.1145/3159652.3159710

\paragraph{Diversifying Complementary Product Recommendation}

Although complementary recommendations are intended to be distinct from the query item, this does not necessarily mean that recommended complements will be distinct from \emph{each other}.
For example, it would presumably not be useful to recommend only t-shirts as a complement for a pair of jeans; instead, one might wish to recommend a combination of shirts, belts, shoes, etc.

Although we'll revisit the notion of diversity in depth in \Cref{chap:ethics}, here we discuss some approaches that consider the problem within the specific context of complementary item recommendation.

\cite{he2016learning} considered that a good set of complementary items might be represented as a \emph{mixture} over different notions of compatibility. They start with a simple pairwise compatibility model between items $i$ and $j$ of the form
\begin{equation}
    d_c(i,j) = \| \gamma_i - \gamma^{(c)}_j \|_2^2
    \label{eq:compEq}
\end{equation}
where $\gamma_i$ and $\gamma^{(c)}_j$ are based on image embeddings. This is similar to the model of \Cref{eq:antonModel}, though includes \emph{separate} embeddings $\gamma$ and $\gamma^{(c)}$ for the `query' item $i$ and complementary item $j$; using separate embeddings breaks the symmetry between $i$ and $j$, which is desirable for complementary items (e.g.~an item should not be complementary with itself).\footnote{As such, \Cref{eq:compEq} is no longer a distance function.}

Next, \cite{he2016learning} notes that \Cref{eq:compEq} captures only a single notion of compatibility; if a model is trained using such a function, this will presumably correspond to the predominant mode of compatibility in the data, but will not be diverse.  To address this, \cite{he2016learning} proposes treat the compatibility relationships in the data as a probabilistic mixture over several competing notions.
This idea borrows from a mathematical framework known as a \emph{mixture of experts} \cite{jacobs1991adaptive}.
Specifically:
\begin{equation}
  d(i,j) = \sum_{c = 1}^C \underbrace{p(c|i)}_{\mathclap{\text{relevance of compatibility function $c$ to query $i$}}} \cdot d_c(i,j).
\end{equation}
Here, $p(c|i)$ measures which types of compatibility relationships $d_c(i,j)$ are most likely to be relevant for a query item $i$; while written as a probability, this can more simply be thought of as a function that is used to combine compatibility relationships with different weights. Here
\begin{equation}
    p(c|i) = \frac{\exp(\theta_k f_i)}{\sum_{k'} \exp(\theta_k f_{i'})},
\end{equation}
where $f_i$ is a feature vector describing the image $i$, and $theta_k$ is a parameter vector.

Ultimately, this model learns $C$ separate embeddings $\gamma^{(c)}_i$ for each item (along with the query embedding $\gamma_i$), each corresponding to a different notion of compatibility or `complementarity.' In principle, this means the model is able to capture several different modes of compatibility that interact simultaneously. At test time, diverse lists of compatible
%\todo{I use the terms compatible and complementary inconsistently}
items can generated by sampling from different compatibility functions $d_c(i,j)$ according to their relevance $p(c|i)$.

\paragraph{Incorporating Product Types}
%\todo{This paper isn't very interesting, could drop}
\cite{citation-0} noted that both accuracy and diversity of complementary product recommendation can be achieved my making explicit use of available category data. Instead of directly predicting which items $j$ are compatible with a query item $i$, the approach first attempts to estimate which of several \emph{categories} are relevant to a given query; following this, the method generates complements via several category-specific compatibility functions, similar to $d_c(i,j)$ form \Cref{eq:compEq}.
%\todo{Figure out venue and replace by actual citation}

%P-Companion: A Principled Framework for Diversified
%Complementary Product Recommendation
%https://assets.amazon.science/d5/16/3f7809974a899a11bacdadefdf24/p-companion-a-principled-framework-for-diversified-complementary-product-recommendation.pdf

\subsection{Implementing a Compatibility Model in \emph{Tensorflow}}
%\todo{Check italics of tensorflow}

Most of the compatibility models we saw in the previous section are relatively straightforward to implement on top of pre-trained image features.

Below we assume a feature matrix $X$ such that $X_i$ is an image feature describing item $i$ (e.g.~a feature from ImageNet as in \cite{he2015vbpr}), and that each pair $(i,j)$ is associated with a label $y$ determining whether the pair is compatible ($y=1$) or not ($y=0$).
%\todo{X_i or x_i?}
The model then projects the images into `style space' via $\gamma_i = E_i x_i$ and $\gamma_j = E_j x_j$; here we use two separate embeddings so that the model can learn asymmetric relationships.
%\todo{Doesn't match any of the notation from when I intorduced the model}
Finally compatibility is evaluated via $\sigma(c - d(\gamma_i, \gamma_j))$:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
class CompatibilityModel(tf.keras.Model):
    def __init__(self):
        super(CompatibilityModel, self).__init__()
        # Embeddings for the query item (Ei) and target item (Ej)
        self.Ei = tf.Variable(tf.random.normal([featDim,styleDim],stddev=0.001))
        self.Ej = tf.Variable(tf.random.normal([featDim,styleDim],stddev=0.001))
        # Offset term as in %\Cref{eq:antonModel?}%
        self.c = tf.Variable(0.0)

    # Given image features x1 and x2, and label y (0/1)
    def call(self, xi, xj, y):
        # Style-space embeddings %$\gamma_i$ and $\gamma_j$%
        gammai = tf.matmul(xi, self.Ei)
        gammaj = tf.matmul(xj, self.Ej)
        # Shorthand for %\Cref{eq:?}%
        return -tf.math.log(tf.math.sigmoid(self.c - tf.math.squared_difference(gammai,gammaj))*(2*y - 1) - y + 1)

model = CompatibilityModel()
\end{lstlisting}
\end{minipage}
%\todo{fill in equation}
%\todo{Change so that the number of style dimensions is a parameter to initialize the model}
%\todo{Squared difference looks wrong, since it's an elementwise operation}

Similarly, we could modify the code to use the compatibility function of \Cref{eq:wangChengPaperEq}:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def call(self, x1, x2, y):
    s1 = tf.matmul(x1, self.E1)
    s2 = tf.matmul(x2, self.E2)
    return -tf.math.log(tf.math.sigmoid(self.c - tf.matmul(s1,tf.transpose(s2)))*(2*y - 1) - y + 1)
\end{lstlisting}
\end{minipage}

Finally, we train the model by sampling compatible and incompatible pairs from our training set and computing gradients:\footnote{Although this example is simple enough to allow reasonably fast training, there are several ways that this code could be made more efficient, e.g.~the embeddings of all images in $X$ could be computed simultaneously.}
%\todo{example in footnote is vague}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def trainingStep(compat):
    with tf.GradientTape() as tape:
        (i1,i2,y) = random.choice(compatiblePairs)
        x1,x2 = X[i1],X[i2]
        objective = model(x1,x2,y)
    gradients = tape.gradient(objective, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
\end{lstlisting}
\end{minipage}

\section{Personalized Generative Models of Images}

In \Cref{chap:text}, we examined personalized text models from two directions: first, we used text within \emph{predictive} tasks, e.g.~we saw how text can be used for regression problems (\Cref{sec:bagOfWords}), and to improve the performance of recommender systems (\Cref{sec:textBasedRecommenders}). Second, we saw how to personalize \emph{generative} models of text (\Cref{sec:personalizedGenerativeText}), i.e.,~to generate text that matches a user's writing style or preferences.

Likewise, our discussion of visual data has so far considered using images to improve predictive performance; it is worth spending a little time exploring how to personalize \emph{generative} models of images.

Briefly, the basic framework we will consider extending is that of the \emph{Generative Adversarial Network}, or GAN.

%\todo{taken from my own paper, maybe change text a bit more}
Generative Adversarial Networks (GANs) are an unsupervised learning framework in which two components `compete' to generate realistic looking outputs (in particular, images) \cite{goodfellow2014generative}. One component (a generator) is trained to generate images, while another (a discriminator) is trained to
distinguish real versus generated images. Thus the generated
images are trained to look `realistic' in the sense that they are
indistinguishable from those in the dataset.

%\todo{same with this paragraph}
Such systems can also be conditioned on additional inputs, in order to sample outputs with certain characteristics \cite{mirza2014conditional}. 

\begin{figure}
%tikz code from https://github.com/PetarV-/TikZ/blob/master/Generative%20adversarial%20network/README.md
\centering
\begin{tikzpicture}
[
ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        thick,
        },
]

	\node[circle, draw, thick] (z) {$\vec{z}$};
	\node[circle, draw, thick, right=5em of z] (x) {$\vec{x}_{\mathit{gen}}$};
	\draw[-stealth, thick] (z) -- node[above] {$G(\vec{z})$} node[below] {generator} (x);
	\node[left=of z] (i) {};
	\draw[-stealth, thick] (i) -- node[above] {$p_\theta(\vec{z})$} (z);
	\node[above=of x, circle, draw, thick] (xt) {$\vec{x}_{\mathit{real}}$};
	\node[left=5em of xt] (it) {};
	\draw[-stealth, thick] (it) -- node[above] {$p_{\mathit{data}}(\vec{x})$} (xt);
	\node[circle, draw, thick, right=5em of x, yshift=2.5em] (D) {$\vec{x}$};
	\node[right=7em of D] (out) {real?};
	\draw[-stealth, thick] (D) -- node[above] {$D(\vec{x})$} node[below] {discriminator} (out);
	
	\node[below=of i, yshift=1em] (dotted1) {};
	\node[below=of out, right=of dotted1, xshift=25em] (dotted2) {};
			
	\node[right=2.5em of x, circle, fill, inner sep=0.15em] (pt1) {};
	\node[right=2.5em of xt, circle, fill, inner sep=0.15em] (pt2) {};
	
	\draw[dashed] (dotted1) -- (dotted2);

	\draw[dashed, thick] (pt1) edge[bend left] (pt2);
			
	\node[circle, draw, thick, fill=white, inner sep=0.15em] at ([xshift=-0.85em, yshift=3.9em]pt1.north) (pt3) {};
			
	\draw[-stealth, thick] (x) -- (pt1);
	\draw[-stealth, thick] (xt) -- (pt2);
	\draw[-stealth, thick] (pt3) -- (D);
	
	%\node[below=of pt1, circle, fill, inner sep=0.15em] (pt3) {};
	\node[below=6em of D, circle, draw, thick] (pt3) {};
	\node[right=0.5em of pt3, circle, fill, inner sep=0.15em] (pt4) {};
	\node[right=0.5em of pt4, circle, draw, thick] (pt5) {};
	
	\draw[-, ArrowC2] (x) |- node[above, xshift=3em] {hello} node[below, xshift=3em] {$\Phi(x_{gen}$} (pt3);
	\node[right=0.1em of pt5] (gamma) {$\gamma_u$};
	\node[below=of pt4] (out2) {compatibility};
	\draw[-stealth, thick] (pt4) -- (out2);
	
	%\draw[-stealth, thick] (pt4) -- (out2);
%	\draw[-stealth, thick] (z) -- node[above] {$G(\vec{z})$} node[below] {generator} (x);

\end{tikzpicture}
\caption{Basic setup of a Generative Adversarial Network (GAN), and a personalized GAN. The components above the dotted line depict the `standard' GAN setup, in which a generator `competes' with a descriminator to generate images that are indistinguishable from real data. Components below the dotted line are used to develop a `personalized' GAN that generates images compatible with a particular user. \label{fig:GANdiagram}}
%\todo{edit this for this specific case, improve caption}
%\todo{also add \gamma_u to the input to the generator}
\end{figure}

%We follow a similar approach based on activation maximization [11] to generate images that best match a selected user's personal style. GANs have been used for broad applications, though to our knowledge we are the first to apply this architecture to fashion image generation

The basic setup of a Generative Adversarial Network 
%(GAN) 
is depicted in \Cref{fig:GANdiagram}. Here $x_{\text{real}}$ is an image sampled from a dataset, whereas $x_{\text{gen}}$ is a synthetically generated image. The discriminator 
%\todo{Clean up all notation}
$D(x)$ is then given an image $x$, and is responsible for predicting whether it is a sample from the dataset or is a synthetic image. 
%\todo{See if this is too similar to the above text or not}
For image generation, the discriminator is typically be a form of convolutional neural network (CNN), whereas the generator is a series of \emph{deconvolution} operators, essentially operating via a similar principle as the CNN, but in reverse. The generator takes as input a latent code $z$, a random input which allows the generator to produce distinct images ($z$ is essentially a \emph{manifold} which describes patterns of variation in image data so as to capture the variability in the training dataset). The discriminator $D$ and generator $G$ are trained simultaneously, such that the generator gradually becomes better at generating images that are capable of `fooling' a better and better discriminator.

The above type of architecture has been used to generate various types of realistic images, including artwork, clothing, and human faces.
%\todo{could use citations but can't find useful ones. Useless sentence anyway}
\cite{kang2017visually} sought to develop \emph{personalized} GANs, that would generate images that capture the preferences of individual users.

The approach from \cite{kang2017visually} essentially combines the GAN framework with a personalized image preference model, similar to that of Visual Bayesian Personalized Ranking (\Cref{sec:VBPR}). The basic idea is depicted in \Cref{fig:GANdiagram} (bottom). Once a GAN is trained (as above), the image $G(z)$ generated by a given latent code is passed to both the discriminator ($D$) as well as a personalized preference model. For the preference model, the image is represented via $\Phi(G(z))$; this is analogous to $\gamma_i$, or $\mathbf{E}f_i$ as in \Cref{eq:vbprEq?}, though unlike VBPR the synthetic image is not associated with any particular item $i$. A user's preference toward this synthetic item is then estimated via $\gamma_u \cdot \phi(G(z))$. Ultimately, the objective is that a generated image should be simultanously plausible (according to $D(G(z))$), but also desirable to the user (according to $\gamma_u \cdot \Phi(G(z))$):
\begin{equation}
    \argmax_z \underbrace{\gamma_u \cdot \Phi(G(z))}_{\mathrlap{\hspace{-3mm}\text{user preference toward generated image}}} - \overbrace{\eta(D(G(z)) - 1)^2}^{\mathllap{\text{`plausability' of generated image}\hspace{-3mm}}}.
    \label{eq:WCGAN}
\end{equation}
%In practice, since the image which optimizes \Cref{eq:WCGAN} is no longer simply an output 

\cite{kang2017visually} argues that a model such as that above can be used in several ways. Most straightforwardly, it can be used to generate designs (or images) that match the preferences of individual users; \Cref{eq:WCGAN} can be straightforwardly modified to find optimal designs for a \emph{population} of users (e.g.~by taking an average $\sum_u \gamma_u \cdot \Phi(G(z))$). Alternately, given an existing image (rather than a generated image $G(z)$), \Cref{eq:WCGAN} (or rather, its gradient) can be used to suggest \emph{local modifications} to the image that will make it preferable to a user or population.


\section{Personalized Image Search and Retrieval}
%\todo{Put this section before or after recommendation?}

%Before visiting visual data in the context of recommendation, it is worth briefly considering how visual data is handled in `traditional' settings like image search.
Having studied the use of visual data in the context of recommendation, it is worth briefly considering how visual data is handled in `traditional' settings like image search and retrieval.


Paper: Image ranking based on user browsing behavior
% http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.707.502&rep=rep1&type=pdf

\cite{trevisiol2012image}


%Paper: Click-through-based cross-view learning for image search

A basic concept in image retrieval settings, as we saw in \cite{sec:SameThingButWithText}, is that of learning a joint embedding space between a query $q$ and an image $i$:
\begin{equation}
%Sec 3.2 of that paper
    d(q,i) = \| g(q) - g(i) \|_2^2
    \label{eq:imageRetrieval1}
\end{equation}
For example, in \cite{pan2014click}, $f(q)$ and $f(i)$ were based on simple linear embeddings of (textual) query features $f_q$ and (visual) image features $f_i$:
\begin{equation}
    g(q) = f_q W^{(\text{query})}; \quad g(i) = f_i W^{(\text{image})}.
\end{equation}
Note that this setting is very similar to that of \Cref{sec:hengelPaper}, in which a query image is linearly projected into `style space' (\Cref{eq:hengelPaper1}) so that neighboring images can be retrieved. This type of image retrieval system operates on the same principle, except that the query features are extracted from text (as we studied in \Cref{sec:textQueryFeatures}).

Finally, $W^{(\text{query})}$ and $W^{(\text{image})}$ are trained so that distances in \Cref{eq:imageRetrieval1} are minimized based on click-through data, again following a learning approach similar to that from \Cref{sec:hengelPaper}. That is, distances should be small between query/image pairs associated with a large number of clicks.\footnote{Note that this approach assumes that query and click data is available at training time, presumably from a method \emph{not} based on visual embeddings.}





Paper: Learning to personalize trending image search suggestion

Paper: Prism: Concept-preserving social image search results
summarization (maybe not...)


\section{History and Related Work}

Personalized recommendation problems involving visual data have been studied for several years. Initial attempts ignore visual data altogether, and long predate solutions that model images explicitly. For example, an early system for clothing recommendation \cite{hu2014style} learns a user's `style' in order to recommend clothing, but does so using `likes' rather than any analysis of visual features. Likewise, \emph{YouTube's} early recommendation approaches \cite{davidson2010youtube} are based on heuristic `relatedness scores' based on co-visitation (essentially a form of neighborhood-based approach, as in \Cref{sec:neighborhood}), though some features based on video metadata are included in the model; newer solutions (based on deep learning) adopt more complicated candidate generation and ranking strategies, though again make little if any use of explicit visual signals \cite{covington2016deep}.

\section{Exercises}
%\todo{Sometimes called problems, sometimes called exercises?}

\begin{exercises}
\item Starting from the code from \Cref{sec:imageComCode}, and using a small dataset of compatible (and non-compatible) pairs (e.g.~from \emph{Amazon Baby} products), set up a pipeline for estimating compatibility relationships (e.g.~`also bought' or `also viewed' products). 
%\todo{style}
Tune the model (e.g.~in terms of the number of embedding dimensions)
%\todo{any other tuning?}
and measure its accuracy (in terms of its ability to successfully distinguish compatible from non-compatible items).
\label{ex:vis1}
\item The model from \Cref{ex:vis1} is based on a distance (or similarity) function of the form
$d(i,j) = \| \gamma_i - \gamma'_j\|_2^2$. Note that there is nothing particularly special about this specific choice of similarity function. 
%\todo{Check whether that's correct}
\item 
%Harder ways of sampling negative items
A challenging aspect of learning a compatibility model between images is that of generating negative samples (i.e.,~pairs of items believed to be incompatible)
\item Although models based on visual features are useful in cold-start settings, there is in principle no reason that 
%\todo{finish}
\item When studying notions like fashion compatibility (or indeed, any latent item representation in a recommender system), it is worth exploring whether the learned representations semantically correspond to our intuitive notion of similarity. To assess this, it helps to visualize latent item representations $\gamma_i$ in two dimensions (for the sake of plotting them). In \Cref{fig:item2vec} we did this simply by learning two-dimensional item representations, though by doing so we are likely visualizing a suboptimal model.
%\todo{Could move visualization outside of this exercise?}
Various techniques exist 
%https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html

\cite{mcinnes2018umap}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
import numpy as np
from sklearn.manifold import TSNE

X_embedded = TSNE(n_components=2).fit_transform(X) # X is a matrix of all item representations %$\gamma_i$%
\end{lstlisting}
\end{minipage}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/item2vecTSNE.pdf}
\caption{Ten-dimensional item representations ($\gamma_i$) embedded into two dimensions via \emph{TSNE}; this is a ten-dimensional version of the model from \Cref{fig:item2vec}. %Representations for items from three distinct categories are shown. 
\label{fig:item2vecTSNE}}
\end{SCfigure}
%\todo{reference figure, write TSNE exercise}

%\todo{Transformers from Scratch -- look into}
%\todo{Auralist - introducing serendipity into music recommendation -- look into}

\end{exercises}
%\todo{These questions are more of a project, could rewrite as a project rather than exercises}

\stepcounter{projectCounter}
\label{proj:visualProj}
\subsection{Project \theprojectCounter: Generating Compatible Outfits}
%\todo{Rename project}

First, consider how you would generate a training dataset of compatible items. It is probably most realistic to start by considering \emph{pairwise} compatibility, i.e.,~to generate a training dataset of pairs of items $(i,j)$ that are mutually compatible (as in \Cref{sec:estimatingCompatFromCopurch} or \Cref{sec:compatTensorflow}). Even then, several options are available for mining pairwise compatibility data. For example:
\begin{itemize}
    \item Co-purchase relationships (e.g.~`people who bought $i$ also bought $j$'), as in \cite{?}.
    %\todo{how do I normally write this?}
    \item Directly mining the co-purchased items from user interaction histories (e.g.~if a user $u$ purchased both items $i$ and $j$, this is an indication that they might be compatible). This strategy was also explored in \cite{?}.
    \item Both of the above approaches are highly noisy, as items are not necessarily co-purchased with the intention of being worn together. A third approach consists of mining explicit relationships from actual outfit data. e.g.
    %\todo{finish, select appropriate paper}
\end{itemize}

Consider the advantages and disadvantages of each of the above approaches. For example, which will allow you to collect the most data, and which will be the least noisy? Further consider how you should select samples, e.g.~you probably want to avoid pairs $(i,j)$ where both items belong to the same category, or might further restrict your dataset to certain categories of interest.

Similarly, you should choose an appropriate strategy to generate \emph{negative} samples for training, i.e.,~pairs $(i,j)$ that do \emph{not} go together. Trivially, such samples could be generated from pairs of random items, though more `difficult' negatives could be generated by selecting pairs with specific categories.

Having built your dataset, there are several potentially interesting directions for study. For example:
\begin{itemize}
    \item What is an appropriate model to use to estimate compatibility relationships? A good starting point may a model such as that from \Cref{eq:asymmetric?}, since compatibility relationships are likely to be asymmetrical in this context.
    %\todo{Make sure this is sufficiently distinct between compatibility vs. outfits}
    \item Consider whether it is worthwhile to incorporate visual features to estimate compatibility via an embedding strategy (e.g.~following the code from \Cref{sec:estimatingCompatibility}) or whether it is sufficient to model compatibility in a latent space (e.g.~as in \Cref{eq:?}).
    \item Consider whether it is useful to incorporate other features, such as brands, prices, features from text, etc.
    \item (harder)
    %\todo{Do I have these `hard' / `harder' tags anywhere? Either use them regularly or not at all}
    Is there value to training a \emph{personalized} model for this task, i.e.,~rather than predicting whether a pair of items $(i,j)$ are compatible, can you predict whether $i$ and $j$ are compatible \emph{for a particular user $u$}.
    %\todo{I think this text may be similar to text elsewhere}
    Think carefully whether the identity of the user explains a significant amount of variation in compatibility relations, and whether enough data can be mined to fit a personalized model.
    %\todo{did I already talk about such a model in the chapter?}
\end{itemize}

Finally, consider ways to visualize the model (or its predictions), either by representing items in a low-dimensional space (as in \Cref{exer:TSNE}), or by building a simple interface to explore compatible items. 

Note that other than the use of visual features, the above steps could be used to build item-to-item compatibility models for \emph{any} types of data (e.g.~dishes in a menu, songs in a playlist), and are not limited to outfit generation.



\chapter{The Ethics of Personalized Machine Learning}
\label{chap:ethics}
%\todo{Change chapter name}

\section{How Can Personalized Machine Learning be Improved?}

So far, we have largely viewed personalized machine learning as a `black-box' machine learning task. That is, given a user, their context, and some potential stimulus, can we estimate how the user will react to that stimulus?

This black-box view of machine learning, while effective for building accurate models, ignores the potential real-world consequences of how such models are applied.

Broadly, the dangers of blindly applying machine learning models are well-studied: ML algorithms can perpetuate, mask, or amplify biases in training data, or have low accuracy for underrepresented groups. Detecting and mitigating these types of biases largely describes the study of `fairness' in machine learning (see e.g.~\cite{dwork2012fairness}).

Within the context of \emph{personalized} machine learning, black-box models, if applied carelessly, can also hide or amplify biases or other issues. Below we highlight a few examples, to be studied throughout this chapter:
\begin{itemize}
    \item A recommender system, although ostensibly designed to aid \emph{discovery}, may actually have a `concentration' effect, where users are gradually locked into a `filter bubble' containing only a narrow set of items (\Cref{sec:filterBubble}).
    \item Alternately, by recommending content that maximally aligns with a user's interests, a system may gradually push them toward more and more `extreme' content (\Cref{sec:filterBubble}).
    \item Recommender systems may have reduced utility for users (or groups of users) who are underrepresented in the training data, for instance `popular' items that are widely recommended may merely reflect what is popular among the majority group (\Cref{sec:?}).
    \item Recommendations may focus only a user's predominant interest, while failing to capture the diversity of breadth of their interactions (\Cref{sec:diversity?}).
    \item Likewise, systems could disadvantage vendors (or content creators, etc.)~by failing to recommend products in the long tail (\Cref{sec:?}).
\end{itemize}

As an example to conceptually demonstrate some of the above issues, \Cref{fig:InnerNN} highlights the ways in which a recommender system, if applied na\"ively, may lead to a `concentration' or `extremification' effect. At left we show personalized recommendations generated by maximizing an inner product ($\gamma_u \cdot \gamma_i$), as in \Cref{sec:modelbased}; at right we show item-to-item recommendations generated by finding similar items (i.e.,~nearest neighbors of $\gamma_i$).

When maximizing an inner product (\Cref{fig:InnerNN}, left), the recommended items are on the `fringe' of the item space; roughly speaking, if I know that a user likes action movies (for example), then I might recommend movies with \emph{the most} action. While this might make sense in the context of movie recommendation, when recommending (e.g.)~political videos on \emph{YouTube}, such a strategy may drive users toward fringe or `extreme' content. Alternately, when choosing nearest neighbors (\Cref{fig:InnerNN}, right), users' recommendations are concentrated around very similar content, which may lead to a `filter-bubble' effect.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/recPointsInner.pdf}
    \includegraphics[scale=0.75]{fig/recPointsNN.pdf}
    \caption{Recommendations selected for a user by maximizing an inner product (left) or taking nearest neighbors (right). \label{fig:InnerNN}}
\end{figure}
%\todo{nowhere to put this}

While the above is merely a conceptual demonstration, in the following sections
%\todo{check organization}
we'll 
%start by 
examine empirical studies (e.g.~from \emph{YouTube} and \emph{Facebook})
%\todo{style}
that analyze filter bubbles and extremification in the context of deployed recommendation settings. 
%
%Later we'll look at issues of 
%\done{Nothing more to add?}
We'll also look further into issues of diversity, bias and fairness, exploring a wide variety of potential consequences of personalized model training.

These ideas connect to the broader topic of fairness and bias in machine learning, though the issues in personalized settings are quite different. Much of our focus when introducing these issues is to present strategies to address them, in order to build personalized models that are more diverse, unbiased, and fair.
%\todo{ugly paragraph, not well connected}

%\section{Diversity}
%\todo{Probably drop section since other sections on diversity are not subsections of this one}

\section{Measuring Diversity}
%\todo{Probably not a subsection}

Before exploring case studies measuring the effect of recommendations on diversity, and techniques to improve diversity, we first briefly consider the kind of measurements we'll use to assess diversity. We'll look at two main concepts: First,
%\todo{Capital after : ?}
across \emph{all} users, do recommended items follow the same \emph{distribution} as the set of items that were consumed? Second, among individual users, are the recommended items more or less diverse than their historical consumption trends?

We first train a recommender (in this case using comic books from \emph{Goodreads}), following the code presented in \Cref{sec:codeImplicit} (i.e.,~using a model based on Bayesian Personalized Ranking).

Next we generate a set of example recommendations from the model and compare those with the original interaction data. For each user, we generate as many recommendations as they have interactions in the original data, so that each user is represented the same number of times in both our interaction data and our recommendation data:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
countsPerItem = defaultdict(int)

for u in range(nUsers):
    # Given a matrix of interactions X, as in %\Cref{sec:implicit?}%
    recs = model.recommend(u, X, N = len(itemsPerUser[u]))
    for i, score in recs:
        countsPerItem[i] += 1
\end{lstlisting}
\end{minipage}
%\todo{Fix cref above}

Next we compare the measurements above to the same measurements derived from the interaction data. In particular, do popular items (based on the number of historical interactions) frequently appear among recommendations; and conversely, are frequently recommended items popular? We plot these comparisons in \Cref{fig:recDiversityComics}.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/diversityDist.pdf}
    \includegraphics[scale=0.75]{fig/diversityDist2.pdf}
    \caption{Distribution of interactions compared to recommendations (based on an implicit-feedback model trained on comic books from \emph{Goodreads}. The left plot measures the \emph{recommendation} frequency of the 200 most popular items (as measured by the number of interactions in the training set); the right plot measures the \emph{interaction} frequency of the 200 most recommended items.}
    \label{fig:recDiversityComics}
    %\todo{Plots could both have different titles}
    %\todo{These plots don't look right. Shouldn't the mass of the compared distributions be the same?}
\end{figure}

Surprisingly, neither of the two appear to match very closely (i.e.,~popular items do not get recommended frequently, and \emph{vice versa}).
%\todo{Do I use the term vice versa ever? Yes several times. Italicize consistently}

There are various ways we could formally measure the discrepancy between these two distributions, or compute summary statistics that help us to compare them.
In the context of item recommendations, we might be interested in whether one of the two distributions is more \emph{concentrated} than the other, i.e.,~whether recommendation frequencies are highly `peaked' around a few popular items (versus a flatter, or longer-tailed distribution).
One measure (that we'll see used in several of the studies below) 
%\todo{check that it's below}
is the \emph{Gini coefficient}, a measure of statistical dispersion. 
%of item interaction frequencies.
Given a set of measurements $y$ (in this case, frequencies associated with each item), the Gini coefficient measures the \emph{average (absolute) difference between frequencies}, i.e.,:
\begin{equation}
    G(y) = \frac{\sum_{i = 1}^N \sum_{j=1}^N | y_i - y_j |}{2 N^2 \bar{y}}.
\end{equation}
%\todo{equation is ugly}
Highly concentrated data will have a large coefficient, while flatter distributions will have $G(y)$ close to zero
(the presence of $2\bar{y}$ in the denominator scales the expression to be in the range $\lbrack 0, 1 \rbrack$).
%\todo{handwavy explanation}

In practice the coefficient is usually computed approximately, i.e.,~by sampling rather than enumerating all possible pairs of items:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def gini(y, samples=1000000):
    m = sum(y) / len(y) # average
    denom = 2 * samples * m
    numer = 0
    for _ in range(samples):
        i = random.choice(y)
        j = random.choice(y)
        numer += math.fabs(i - j)
    return numer / denom
\end{lstlisting}
\end{minipage}

In the case of this particular experiment, the interaction data yields a Gini coefficient of $G = 0.72$ while the recommendations yield $G = 0.80$.
%0.7201648536364775
%0.80445217092293
In other words, this particular recommendation algorithm has resulted in recommendation that are somewhat more concentrated compared to historical interaction data.

\subsection{Filter Bubbles and Extremification}
\label{sec:filterBubble}

\cite{pariser2011filter} (coins the term filter bubble)
%see https://www2.kbs.uni-hannover.de/fileadmin/institut/pdf/webscience/2016-17/papers/nej1.pdf
% for reading list on filter bubble background

The idea that personalized recommendation traps users in `filter bubbles,' amplifies existing biases toward popular items, or guide users toward extreme content, are often reported in popular media and discussed anecdotally, though such concepts are often not precisely defined. It is also difficult to measure these types of dynamics empirically, as one rarely has the ability to analyze the counterfactual scenario in which the recommender wasn't in place.
%\todo{Ugly opening paragraph}

%Paper: Recommender Systems and their Impact on Sales Diversity

\paragraph{Exploring Diversity through Simulation}

An early paper that attempted to define and analyze the impact that recommender systems have on interaction diversity did so via simulation \cite{fleder2009blockbuster}. They noted the existence of two competing hypotheses as to why recommender systems might encourage or discourage diversity: on the one hand, recommender systems can guide content \emph{discovery}, which can increase the diversity of item interactions \cite{brynjolfsson2006niches}; on the other hand, recommender systems might reinforce the popularity of already-popular products, thus reducing diversity \cite{mooney2000content}. Their attempt to resolve this question built a simple simulation which generates recommendations, 
in which the probability of an item being recommended, or the probability of a user accepting it, can be controlled.
%which simulated users have a controlled probability of accepting. 
%By controlling the acceptance probability (or other properties of the simulation), they can simulate recommendations with varying levels of influence.
By varying the controllable parameters, they show that under almost all conditions (i.e.,~except in edge cases), the recommender system leads to a concentration effect (i.e.,~results in a reduction in interaction diversity), as measured by the Gini coefficient.
%(a measure of the statistical disbursion of item interaction frequencies).
%MOVED UP

Of course, the above is not necessarily true of \emph{every} recommender system; indeed as we'll see in \Cref{sec:?}, one can design a recommender system so as to explicitly target the diversity of recommended items.
Rather, the above result is simply a demonstration that under fairly minimal conditions, recommender systems \emph{can} lead to a concentration effect.

\paragraph{Directly Measuring Algorithmic Recommendation and Diversity}
%\todo{Crappy section title}

Following \cite{fleder2009blockbuster}, which studied the possibility of filter bubbles via simulation, \cite{nguyen2014exploring}
was the first to empirically measure the effect of recommender systems on content diversity in a real setting.

The research questions in \cite{fleder2009blockbuster} are similar to the ones studied via the simulations above, namely: do recommender systems gradually expose users to narrower content over time; and, how does this effect vary as a function of how receptive users are to recommendations.
%\todo{finish, tag genome, etc....}

\paragraph{Exploring Diversity through \emph{YouTube} Recommendations}
%\todo{Dunno about these paragraph or overall organization}

%
%Paper: The impact of YouTube recommendation system on video views .
%
Following this work, \cite{zhou2010impact} conducted an empirical study of recommendations on \emph{YouTube},
%\todo{Rewrite the above sentence, since I've reorganized}
%
%For example, an early paper studying \emph{YouTube} recommendations argued that \emph{YouTube's}
%\todo{italics?}
and argued that \emph{YouTube's} recommendations (specifically the `related videos' feature) have a positive impact on content diversity.
They showed that recommendations drive a large fraction of views on youtube,
%\todo{captilaitaion and italicize}
and that views driven by recommendations have higher diversity than views from a popularity-driven system (however, it is hard to argue that recommendation-driven views are therefore `diverse,' given that popularity-driven views would presumably lead to high concentration, and as such are arguably a weak baseline).
%\todo{not beautifully worded}




\paragraph{Diversity and Extremification}

However, even if consumption is `diverse,' as defined by the overall distribution of interactions across an item vocabulary, this does not guarantee that users are necessarily exposed to diverse viewpoints, or are not pushed toward extreme content.
%Let's consider the trivial motivating example in \Cref{fig:InnerNN}
%\todo{Could introduce it here}
For example, returning
%\todo{is it cited before?}
to the motivating example from \Cref{fig:InnerNN}, recommendations based on maximizing an inner product (left) would expose users to more `diverse' content than the nearest neighbor model (right), but may gradually drive users toward `fringe' items that are aligned with, but are more extreme versions of, the content they initially consumed.

\cite{ribeiro2020auditing} attempted to empirically analyze the pathways via which users arrive at extreme content on \emph{YouTube}. 
%\todo{Finish}
The authors used curated lists of channels (for their study, of alt-right political channels) in order to establish a ground-truth of what is meant by `extreme' content; they also collected less extreme content (`alt-lite,' general media, etc.)

%Auditing Radicalization Pathways on YouTube (https://arxiv.org/pdf/1908.08313.pdf); separate section on extremification?

\section{Diversification Techniques} % Filter bubble
\label{sec:diversification}

% https://www.sciencedirect.com/science/article/abs/pii/S0950705117300680



one can imagine how recommendations based on nearest neighbors or maximum inner-products, as in \Cref{fig:InnerNN}, might focus exclusively on a user's predominant interest alone.


Bias Disparity in Recommendation Systems

Diversifying Search Results (Agarwal)


%\todo{Move diversity to recommender systems chapter?}

\subsection{Maximal Marginal Relevance}
\label{sec:MMR}
%\todo{I think I found this from a DPP paper, probably the youtube one. Maybe check the argument they make relating this to other ideas}

A simple notion of diversity that is commonly used in document retrieval scenarios is that among a (ranked) list of retrieved documents, each retrieved item should simultaneously be relevant, but at the same time not too similar to the already-returned items.

This notion is captured by the \emph{Maximal Marginal Relevance} \cite{carbonell1998use} procedure (MMR). The approach was originally designed for retrieving sets of text passages that best summarize a document (with respect to some query): each document should be similar to the query, but also dissimilar from the documents already retrieved.

The same concept can straightforwardly be applied in recommendation scenarios, given that we have notions of both \emph{relevance} and \emph{similarity} available (e.g.~relevance might be the output of a latent factor model, while similarity could be defined in terms of cosine similarity, or as an inner product between item representations $\gamma_i$ and $\gamma_j$ for two products).

To apply the concept to recommendation, we would define the Maximal Marginal Relevance as follows:
\begin{equation}
    \mathit{MMR} = \argmax_{i \in R \setminus S} \Bigl[ \lambda\underbrace{\mathit{Sim}^{\text{user}}(i,u)}_{\mathclap{\text{relevance to the user}}} - (1 - \lambda)\overbrace{\max_{j \in S}\mathit{Sim}^{\text{item}}(i,j)}^{\mathclap{\text{similarity to already-recommended items}}}\Bigr],
    \label{eq:MMR}
\end{equation}
where $R$ is an initial candidate set of recommendations (most trivially e.g.~a list of items the user hasn't already interacted with), and $S$ is a set of items retrieved so far. $\mathit{Sim}^{\text{user}}$ and $\mathit{Sim}^{\text{item}}$ are item-to-user and item-to-item similarity functions (respectively); the former is presumably the compatibility function returned by a recommender system,
%\todo{repetitive writing compared to above paragraph}
the latter is any item-to-item similarity measure (e.g.~cosine similarity or inner product between item representations).

Note that the above is computed iteratively, that is we add one result at a time by maximizing the MMR until the list $S$ has the desired size. Finally, $\lambda$ trades off the extent to which we care about compatibility versus diversity.

\subsection{Re-ranking Approaches to Diverse Recommendation}
%\todo{Bad section name, since MMR is also reranking}

%Improving aggregate recommendation diversity using ranking-based techniques \cite{adomavicius2011improving}

Similar to Maximal Marginal Relevance, several approaches have been specifically designed for re-ranking in recommendation scenarios. \emph{Re-ranking} approaches, including MMR, assume that we are given an initial ranking function which we trust to find items of high relevance, but which lack diversity; thus one seeks a re-ranking procedure that balances the two concerns.

One such approach for re-ranking for recommendation was proposed in \cite{adomavicius2011improving}. The approach assumes the presence of three components: First, they assume the availability of a compatibility score, e.g.~a rating prediction $r(u,i)$. Second, a \emph{relevance-oriented} ranking technique, $\mathit{rank}_u(i)$; this could be any ranking function, though most trivially one might simply order predictions by a predicted rating score $r(u,i)$. And third, a second `diversity oriented' ranking function; conceptually, this should focus on recommending items to users that they would \emph{not} normally consider.

An example of a diversity-oriented loss from \cite{adomavicius2011improving} is to sort items by popularity, with the \emph{least} popular items being ranked first:
\begin{equation}
    \mathit{rank}^{(\text{pop})} = | U_i |.
\end{equation}
Recommending \emph{unpopular} items does not at first appear to be a particularly effective recommendation strategy; however this ranking is used in conjunction with the prediction score $r(u,i)$. Specifically, to encourage diversity we want to find \emph{unpopular} items that \emph{this} user is likely to enjoy. Spiritually, this is somewhat reminiscent of our \emph{TF-IDF} approach to finding important words in documents (\Cref{sec:tfidf}).
%; here `words' correspond to items, and documents (roughly) correspond to the set of items this user 

The specific (re-)ranking objective from \cite{adomavicius2011improving} then looks like
\begin{equation}
    \mathit{rank}'_u(i, t) = \left\lbrace
    \begin{tabular}{ll}
    $\mathit{rank}^{(\text{pop})}(i)$ & if $r(u,i) \geq t$\\
    $\alpha_u + \mathit{rank}_u(i)$ & otherwise
    \end{tabular}
    \right..
\end{equation}
Here $t$ is a threshold term, essentially determining whether one of the low-popularity recommendations has a high enough score to recommend; $\alpha_u$ is an offset term ensuring that the popularity-based recommendations appear first in the ranking before those of $\mathit{rank}_u(i)$.

\cite{adomavicius2011improving} shows that as the threshold $t$ is changed, the system gradually trades-off between recommendation precision and diversity. 
Several different ranking functions are considered, for example to replace popularity-based ranking by alternatives based on the average rating, rating variance, etc.

They also note that the type of diversity achieved by this ranking mechanism is quite different from that in \Cref{sec:MMR}, as it does nothing to encourage diversity (or dissimilarity) among an individual user's item list. Instead, they discuss the related notion of \emph{aggregate diversity}, which defines diversity across the item vocabulary itself, i.e.,~recommendations across \emph{all} users should have reasonable coverage of the complete item vocabularity. This is related to the notion of $P$-fairness
%\todo{Is this C or P? I think P is right}
that we'll discuss in \Cref{sec:multisided}.

\subsection{Topic Diversification}

Improving recommendation lists through topic diversification \cite{ziegler2005improving}

Avoiding Monotony: Improving the Diversity of Recommendation Lists \cite{zhang2008avoiding}
%can't find pdf

Category-Driven Approach for Local Related Business
Recommendations
%https://dl.acm.org/doi/pdf/10.1145/2806416.2806495

\section{Other Metrics Beyond Accuracy: Novelty, Discovery, and Serendipity}
%\todo{Don't like section title, as diversity is covered in other subsections above and below. CHANGED}
%\todo{probably could go after the DPP section}

So far, we have considered diversity in terms of the trade-off between recommending the highest \emph{relevance} items (e.g.~highest click probability) while ensuring that recommended items are not too similar to each other.
Other than relevance, diversity among items is only one desirable alternative.

Besides relevance and diversity, other desirable features of a recommendation list might include:
\begin{itemize}
    \item Items should be \emph{novel} to the user, i.e.,~the recommender system should balance \emph{discovery} of new items against recommending items with high interaction probability, but which are already known to the user.
    \item Rather than being internally diverse, we might have goals such as mutual compatibility among items, as in \cite{?}.
    %\todo{citation? tong paper maybe?}
    \item Recommended items should have good \emph{coverage}, i.e.,~they should represent a broad range of categories or features; or they should be \emph{balanced}, in terms of matching the category distribution from the user's history (as in \Cref{sec:?}).
    %\todo{check that this is after this}
    %Coverage and balance
    \item Other goals could be more nebulous, such as perceived unexpectedness, serendipity, or overall user satisfaction.
\end{itemize}

\cite{kaminskas2016diversity} broadly surveys these alternate optimization criteria for recommneder systems, focusing in particular on diversity, serendipity, novelty, and coverage. We briefly survey some of their main findings (as well as more recent work) below.
%\todo{finish. What do they say about diversity? Adding a bit below}

\paragraph{Further Perspectives on Diversity}
%\todo{move out of this section into diversity section? Or drop?}
Most of the approaches to diverse recommendation in \cite{kaminskas2016diversity} are focused on \emph{reranking} strategies, similar to maximum marginal relevance (\Cref{sec:MMR}) and other techniques we've discussed so far.
%\todo{finish. what else do they have to say?}
They also discuss other settings where diversity can be useful, such as conversational recommendation (\cite{kelly2006enhancing}, see \Cref{sec:?}),
%\todo{keep ref only if it's covered}
and the relationship to more traditional work in `portfolio optimization' from information retrieval \cite{?}.
%\todo{finish}


\cite{vargas2011rank} and \cite{vargas2011new} studied the relationship between novelty and diversity when making recommendations, and presented several metrics to evaluate existing systems in terms of such characteristics.
%\todo{drop?}

\subsection{Serendipity}
%\todo{Don't like the subsections/section, keep working on}

Various attempts have been made to define `serendipity' in the context of recommendations. \cite{kaminskas2016diversity} starts with the core property of `surprise' (i.e.,~recommendations should be different from one's expectations); \cite{kotkov2018investigating} state that serendipity should be a combination of relevance, novelty and unexpectedness.

The exact definitions of each of these competing elements is difficult to define

\cite{ge2010beyond}

\paragraph{Discovery}

Discovery-oriented Collaborative Filtering for Improving User Satisfaction (not sure)

\paragraph{Unexpectedness}
%On Unexpectedness in Recommender Systems: Or How to
%Expect the Unexpected 
%
\cite{adamopoulos2014unexpectedness}
%http://ceur-ws.org/Vol-816/paper2.pdf
attempt to define the notion of `unexpectedness' as it relates to recommendations. They note that one cannot target unexpectedness in isolation, or one could trivially generate poor-quality but unexpected recommendations. As such they seek a notion of \emph{utility} that balances unexpectedness 
%must be balanced against 
against traditional metrics of recommendation
quality.
%
They define unexpectedness (for a user $u$ and item $i$) as a distance between $i$ and the set of items the user $u$ `expects' to receive. They further assume that there is some optimal value for this distance (which could be different for each user): recommendations that are too expected are uninteresting, while recommendations that are too unexpected will be regarded as irrelevant (quality is defined more straightforwardly in terms of ratings. 
%
%The goal is then to determine the 

Several quantities must then be determined: each user's personal tolerance for unexpectedness,
the ideal trade-off between unexpectedness and utility, and finally the definition of what is `expected.' 
For the latter \cite{adamopoulos2014unexpectedness} use a definitions based on content similarity in terms of movie attributes (movies with similar attributes are `expected').
%\todo{mention where that it's movies?}
The goal of \cite{adamopoulos2014unexpectedness} is not to fit these values (which are largely subjective quantities) but to evaluate the performance of different recommendation approaches under various hypothetical scenarios. The most promising finding is that optimizing this type of joint utility need not harm performance compared to methods that target quality exclusively.
%\todo{Not a great summary}

%\todo{Would be nice to make these paragraphs and upper paragraph subsection}
\paragraph{Serendipity in Music Recommendation}

\cite{zhang2012auralist} considers how music recommendations can be improved by balancing goals of accuracy, diversity, novelty, and serendipity. Their specific approach combines many of the ideas we've seen already:
%\todo{check ordering}
\emph{diversity} is measured in terms of the cosine similarity between items in a recommendation list (as in \Cref{sec:?}); \emph{novelty} or `unexpectedness' is defined in terms of overall item popularity (as in \Cref{sec:?}); serendipity (or `unserendipity,' since low values mean high serendipity) is defined using a novel function, which essentially measures how similar recommended items are to those in the user's interaction history:
%\todo{finish}
\begin{equation}
  \text{Unserendipity} = \frac{1}{|\mathcal U|}\sum_{u \in \mathcal{U}}
  \frac{1}{| I_u|}\sum_{i \in I_u} \sum_{j \in R_u} \frac{\mathit{Cos}(i,j)}{|R_u|}.
  \label{eq:auralistSerendipity}
\end{equation}
%\todo{check notation}
Where $R_u$ is a set of items recommended to the user (and $I_u$ is the item history for user $u$). This measure takes a low value if recommended items are on average different from those that appeared in the user's history.
%\todo{last sentence somewhat redundant}

Given these three metrics (diversity, novelty, and serendipity), \cite{zhang2012auralist} seeks recommendation techniques that can optimize them without overly compromising accuracy. While metrics such as that of \Cref{eq:auralistSerendipity} cannot straightforwardly be incorporated into the optimization scheme directly, various models are designed 
to ensure that recommendations are topically diverse or belong to distinct clusters. Quantitively, \cite{zhang2012auralist} study the trade-off between accuracy, diversity, novelty, and serendipity under different configurations of this model. They also conduct a user study to evaluate the qualitative aspects of the model, revealing that subjective notions of `subjectivity' and `usefulness' can be improved without overly harming user enjoyment.


\paragraph{Investigating Serendipity via User Studies}

Given the ambiguous nature of the precise definition of serendipity, \cite{kotkov2018investigating} attempts to assess what it means to users via a survey. \cite{kotkov2018investigating} surveys commonly proposed notions for diversity, ranging from 
%items that are simply unfamiliar to a user \cite{kapoor2015like}, to 
items the user simply hasn't heard of, didn't expect to be recommended, or is highly disimilar to what they usually consume.


%\todo{Maybe. Somewhat too complex. Probably couple of pages to write}
% Rank and Relevance
% in Novelty and Diversity Metrics for Recommender Systems \cite{vargas2011rank}

% New Approaches to Diversity and Novelty in
% Recommender Systems \cite{vargas2011new}

PURS: Personalized Unexpected Recommender System for Improving User Satisfaction \cite{li2020purs}
%Maybe, similar to the tf-idf-like paper, maybe talk about in same place



%Investigating serendipity in recommender systems based on real user feedback
%https://www.researchgate.net/publication/326164444_Investigating_serendipity_in_recommender_systems_based_on_real_user_feedback

The Impacts of Item Features and User Characteristics on Users' Perceived Serendipity of Recommendations


\subsection{Determinantal Point Processes}
%\todo{Needs to move to diversity section}
%Youtube papers

In \Cref{sec:MMR} and \Cref{sec:ImprovingRecommendationListsPaper}
%\todo{Others}
we discussed various approaches that attempt to balance accuracy and diversity via what are essentially `heuristic' strategies that greedily select items that maximize utility while being sufficiently novel compared to the rest.
%\todo{Is the above a fair description?}
%\todo{Needs gluing sentence}

Determinantal Point Processes (or DPPs) \cite{kulesza2012determinantal} are a \emph{set-based} optimization technique that can be used to identify subsets of items that simultaneously maximize item quality and diversity among items. Specifically, given a set of items $I$, a DPP assigns a probability $P(S)$ to every subset $S \in \mathbb{P}(I)$, where $\mathcal{P}(I)$ is the \emph{powerset} (i.e.,~the set of all subsets) of $I$. The goal is then to model (i.e.,~parameterize) this probability, either globally or for individual users, such that finding the subset of items which maximizes $P(S)$ has an optimal tradeoff between utility and diversity.

\cite{wilhelm2018practical} studied the application of DPPs to diversify recommendations on \emph{YouTube}.
%\todo{italics, caps}

The method assumes a few inputs. First, as with previous diversification techniques, we assume a utility or `quality' estimate $f(u,i)$ is given (e.g.~from a pre-trained recommender system), which encodes the probabilty that user $u$ will interact with item $i$ given $i$'s features; we also assume a predefined distance function $d(i,j)$ between two items.

Next, we have historical sets of items that have been surfaced to the user (i.e.,~the outputs of an existing system), along with subsets of items that the user selected (i.e.,~binary labels indicated by $y_{u,i}$). The goal is to select subsets of items that will maximize the total number of interactions, which in practice is trained by maximizing the Cumulative Cain:
\begin{equation}
    \sum_{u}\sum_{i} \frac{y_{u,i}}{\mathit{rank}(u,i)},
\end{equation}
where $\mathit{rank}(u,i)$ is the new rank assigned by the proposed algorithm. That is, the items that the user interacted with ($y_{u,i} = 1$) should have high rank.
%\todo{Cover Cumulative Gain in recommender systems evaluation or something}

Note that the above seems reminiscent of `traditional' approaches to recommendation, i.e.,~we are ranking items such that positive interactions should have high rank (which seems similar to what we saw in \Cref{sec:CumGanEval}). The main difference here is simply the observation that the total number of interactions will be maximized when utility and diversity are balanced (e.g.~a user will quickly become bored if recommendations cover only one of their interests).

Next, given a candidate set of $N$ items, we next define a matrix $L^{(u)}$ such that diaginal entries $L^{(u)}_{i,i}$ encode the utility of an item $i$ and off-diagonal entries $L^{(u)}_{i,j}$ encode the similarity of two items $i$ and $j$. The specific parameterization used in \cite{wilhelm2018practical} is:
\begin{eqnarray}
  L^{(u)}_{i,i} & = & f(u,i)^2\\
  L^{(u)}_{i,j} & = & \alpha f(u,i) f(u_j) \exp(- \frac{d(i,j)}{2\sigma^2}) \quad \text{for $i \neq j$}.
  \label{eq:dppParams}
\end{eqnarray}

Now, the quality of a subset $S$ is proportional to the determinant of the submatrix of $L$ induced by $S$, $\det(L_S)$. Specifically:
\begin{equation}
    P(S) = \frac{\det(L_S)}{\sum_{S' \in \mathbb{P}(I)} \det(L_{S'})}.
    \label{eq:dpp1}
\end{equation}
%\todo{Could rewrite everything with subset notation rather than element of powerset}
Critically, the denominator of the above equation can be computed efficiently as
\begin{equation}
    \sum_{S' \in \mathbb{P}(I)} \det(L_{S'}) = \det(L + I),
\end{equation}
where $I$ is the identity matrix.

To understand (roughly) why the determinant is diversifying, it helps to consider the trivial example where $S$ consists of only two items $i$ and $j$; then the determinant is given by
\begin{equation}
    \det \left( \left[ \begin{tabular}{cc}
      $L_{i,i}$ & $L_{i,j}$\\
      $L_{j,i}$ & $L_{j,j}$
    \end{tabular} \right] \right) = L_{i,i}L_{j,j} - L_{i,j}L_{j,i};
\end{equation}
this value will be maximized when the utility is high ($L_{i,i}L_{j,j}$) and the similarity is low ($L_{i,j}L_{j,i}$).

In spite of the relatively simple form of \Cref{eq:dpp1}, it is still not practical to solve the (NP-hard) problem of finding the optimal subset. In \cite{wilhelm2018practical} this is addressed by a simple greedy algorithm (similar to that of \Cref{sec:MRR}, among others), in which one starts with the empty set of videos $S = \emptyset$, and iteratively adds the item $i$ which maximizes the determinant $\det(L_{S \cup \lbrace i \rbrace})$

Note that the parameterization in \Cref{eq:dppParams} includes two parameters, $\alpha$ and $\sigma$, which in practice are tuned using grid search. Intuitively these parameters control the relative weight of utility versus diversity ($\alpha$), and the `tightness' of the similarity function ($\sigma$). These parameters are global, but in principle could be learned per-user.
%\todo_{Have I defined grid search anywhere?}

Ultimately, the experiments find that implementing the above DPP on a user's video feed increases user satisfaction (as measured by session duration), compared to various other diversification strategies.

%Practical Diversified Recommendations on YouTube with Determinantal Point Processes


\subsection{Calibration}

A related notion to diversity is that of \emph{calibration} of predictions or recommentations. Whereas a diversity metric might suggest (for example) that we should expose users to a wide distribution of recommendations, which potentially span beyond their explicit preferences, \emph{calibration} refers to the idea that recommendations should be made \emph{in proportion} to expressed preferences. For instance, if a user watches 40\% sci-fi movies and 60\% romantic comedies, they should not exclusively be recommended romantic comedies, as might happen when na\"ively recommending nearest neighbors or maximizing inner products.

\cite{steck2018calibrated} introduces such a notion of \emph{calibrated recommendations}, in the context of movie recommendations on \emph{Netflix}. Their work discusses both metrics to assess calibration, as well as methods to calibrate the outputs of an existing recommender system.

Their notion of calibration operates over a pre-defined set of item \emph{genres}, described using a stochastic genre vector $p(g|i)$ (e.g.~a movie might be categorized as 80\% `action' and 20\% `sci-fi'); this could potentially be adapted to other attributes toward which one desired calibration. The basic idea behind a calibration metric is then that the distribution of genres among a 
user's history $\i \in \mathcal H$
should match the distribution of \emph{recommended} items $i \in \mathcal I$. 
The two terms are defined (respectively) as:
\begin{eqnarray}
p(g|u) & = \frac{\sum_{i \in \mathcal H} w_{u,i} \cdot p(g|i)}{\sum_{i \in \mathcal H} w_{u,i}}\\
q(g|u) & = \frac{\sum_{i \in \mathcal I} w_{r(i)} \cdot p(g|i)}{\sum_{i \in \mathcal H} w_{r(i)}}.
\end{eqnarray}
Both expressions include a `weighting' term $w$. In the case of the historical distribution $w_{u,i}$ might weight items according to recency (for example), or for the case of recommendations $w_{r(i)}$ might weight recommendations according to their position in a list (i.e.,~their ranking); either term could also be ignored.

Now, the goal is to generate a set of recommended items $\mathcal I$ such that the two distributions should match closely. The difference between the two distributions can be measured by (e.g.)~the Kullback Liebler divergence:
\begin{equation}
    \text{KL}(p,q) = \sum_{g} p(g|u)\log \frac{p(g|u)}{q(g|u)}.
\end{equation}
Of course, the recommendations should also be highly compatible according to the recommender system itself. This is achieved in \cite{steck2018calibrated} with a simple expression that trades off recommendation utility and calibration (via a trade-off hyperparameter $\lambda$):
\begin{equation}
  \mathcal I = \argmax_{\mathcal I} \underbrace{(1 - \lambda) \cdot \sum_{i \in I} s(i)}_{\text{compatibility}} - \underbrace{\lambda \cdot \text{KL}(p, q(\mathcal I))}_{\text{calibration}}
  \label{eq:calibrated}
\end{equation}
\cite{steck2018calibrated} notes that the above is a hard combinatorial optimization problem, but can be approximated greedily (with a certain optimality guarantee), by iteratively adding one item at a time to $\mathcal I$ so as to optimize the above criterion until the desired number of items is reached.

An appealing property of this approach is that it can be applied in a purely post-hoc fashion to the outputs of \emph{any} recommender system that associates scores between users and items. The experiments in \cite{steck2018calibrated} show (by varying $\lambda$ in \Cref{eq:calibrated}) that a reasonable degree of calibration can be achieved with minimal loss in recommendation utility.

%Calibrated recommendations: https://dl.acm.org/doi/10.1145/3240323.3240372



%\todo{Figure needs to go elsewhere}

% \section{Clickbait, Fake News, and Review Fraud}

%  Review fraud:
% BIRDNEST: Bayesian Inference for Ratings-Fraud Detection

% \section{Optimizing Short-Term versus Long-Trem Goals}

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Summary of diversification techniques.}
\label{tab:divTech}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{|XXX|}
\hline
\textbf{Technique} & \textbf{Reference} & \textbf{Description}\\
Maximum Marginal Relevance & \cite{carbonell1998use}, \Cref{sec:MMR} & Recommended items should balance utility against diversity compared to already-recommended items\\
Aggregate Diversity & \cite{adomavicius2011improving} & Recommended items should be those that have high compatibility for a particular user, but low aggregate compatibility (e.g.~popularity); this will lead to \emph{aggregate} diversity of recommendations across the entire population\\
Determinantal Point Processes & \cite{wilhelm2018practical} \Cref{?} & Balance utility and diversity (as with MMR above), but using a set-based objective\\
Calibration & \cite{steck2018calibrated} & Recommendations should exhibit the same distribution of attributes (e.g.~in terms of recommended categories) as users' historical interactions\\
\hline
\end{tabularx}
\end{minipage}
\end{table}
%\todo{Fix, figure out how best to reference sections in table}

\subsection{Implementing a Diverse Recommender}
%\todo{Should this be subsection here or elsewhere?}

Here we'll briefly describe an implementation of a diversified recommender, based on the \emph{maximum marginal relevance} method of \Cref{sec:?} (though other reranking-based methods are similar).
%\todo{generic opening}

We start by building a few utility data structures. First we collect the list of candidate recommendations, excluding any items the user has already consumed. Next we compute compatibility scores between a user $u$ and all items. Here our compatibility scores (i.e.,~$\mathit{Sim}^{\text{user}}(i,u)$ in \Cref{eq:MMR})
are simply the output of a latent-factor recommender
(here we use a batch-based prediction function as in \Cref{sec:fastRec}). We sort these in order from the highest- to the lowest-rated items. In practice we might want to re-rank only the top few hundred items rather than computing diversity scores for extremely low-compatibility items.

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
candidates = list(itemSet.difference(itemsPerUser[u]))
compatScores = list(zip([float(f)
    for f in model.predictSample([userIDs[u]]*len(candidates),
        [itemIDs[i] for i in candidates])], candidates))

compatScores.sort(reverse=True)
\end{lstlisting}
\end{minipage}

Next
we implement a function to determine the similarity between a candidate recommendation and others already in the list (i.e.,$\max_{j \in S}\mathit{Sim}^{\text{item}}(i,j)$ from \Cref{eq:MMR}). 
\texttt{itemEmbeddings} is a lookup table containing embeddings $\gamma_i$ for each item. 
The similarity function (\texttt{sim}) is the cosine similarity (not shown), though other similarity functions could be substituted (including simple alternatives such as checking whether items belong to the same category):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
itemEmbeddings = dict(zip(candidates,
    tf.nn.embedding_lookup(model.gammaI, [itemIDs[i] for i in candidates])))

def maxSim(itemEmbeddings,i,seq):
    if len(seq) == 0: return 0
    return max([sim(itemEmbeddings,i,j) for j in seq])
\end{lstlisting}
\end{minipage}

To implement the iterative reranker
%\todo{rerank or re-rank}
we define a method that takes the list of recommendations generated so far (\texttt{seq}), and generates the next item to be added to the list, based on the weighted combination from \Cref{eq:MRR}. $\lambda$ is passed as an argument to the function to trade-off the importance of compatibility and diversity:

%\todo{Correct all code to have even spacing between arguments "item, user" vs "item,user"}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def getNextRec(model,compatScores,itemEmbeddings,seq,lamb):
    scores = [(lamb * s - (1 - lamb) * maxSim(itemEmbeddings,i,seq), i)
        for (s,i) in compatScores if not i in seq]
    (maxScore,maxItem) = max(scores)
    print(maxScore)
    return maxItem
\end{lstlisting}
\end{minipage}

Note that the above implementation is inefficient and even on a modestly sized dataset (in terms of the size of the item vocabulary) requires several seconds to generate recommendations. Several strategies might be used to improve its performance, including efficient retrieval techniques (as in \Cref{sec:?}), or by exploiting certain structure in our compatibility or diversity functions that would obviate the need to exhaustively compute all scores.

\paragraph{Examples of Diversified Recommendations}
%
\Cref{tab:diverseRec} shows examples of diversified recommendations on  data from \emph{BeerAdvocate}.
%\todo{Do I call it beeradvocate or just the beer dataset?}
Different values of $\lambda$ are chosen to control the compatibility/diversity trade-off (for a randomly chosen user). The first set of recommendations ($\lambda = 1$) optimizes only for compatibility: the user is recommended a selection of rich stouts and IPAs. Decreasing $\lambda$ by a little (middle column) introduces a few `lighter' yet similar beers; decreasing $\lambda$ further results in beers from a wide variety of categories (wheat beers, lambics, scotch ales, etc.).

Note that the ideal value of $\lambda$ depends on a variety of factors, including our specific choices of compatibility and diversity functions (which may not even be on the same scale: in our case one is a rating from (in the range $\lbrack 1, 5 \rbrack$) and the other is a cosine similarity (in the range $\lbrack -1, 1 \rbrack$). The solution can also be sensitive to hyperparameters (e.g.~the number of factors and how strongly we regularize). In practice the optimal amount of diversity may simply be guided by what `looks right.'
%\todo{style}

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Diversified recommendations (maximum marginal relevance).
%\todo{maximum or max? Here and elsewhere}
}
\label{tab:diverseRec}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{lXXX}
% \hrule\\
\textbf{rank} & \textbf{Low diversity} & \textbf{Medium diversity} & \textbf{High diversity}\\
1  & Founders KBS (Kentucky Breakfast Stout) & Founders KBS (Kentucky Breakfast Stout) & Founders KBS (Kentucky Breakfast Stout)\\
2  & Two Hearted Ale                         & Samuel Smith's Nut Brown Ale            & Samuel Smith's Nut Brown Ale\\
3  & Bell's Hopslam Ale                      & Two Hearted Ale                         & Salvator Doppel Bock\\
4  & Pliny The Elder                         & Bell's Hopslam Ale                      & Oil Of Aphrodite - Rum Barrel Aged\\
5  & Samuel Smith's Oatmeal Stout            & Kolsch                                  & Great Lakes Grassroots Ale\\
6  & Blind Pig IPA                           & Drax Beer                               & Blue Dot Double India Pale Ale\\
7  & Stone Ruination IPA                     & A Little Sumpin' Extra!~Ale             & Calistoga Wheat\\
8  & Schneider Aventinus                     & Odell Cutthroat Porter                  & Dogwood Decadent Ale\\
9  & The Abyss                               & Miner's Daughter Oatmeal Stout          & Traquair Jacobite\\
10 & Northern Hemisphere Harvest Wet Hop Ale & Rare Bourbon County Stout               & Cantillon Gueuze 100\% Lambic
%
% Founders KBS (Kentucky Breakfast Stout)
% Samuel Smith's Nut Brown Ale
% Two Hearted Ale
% Bell's Hopslam Ale
% Kolsch
% Drax Beer
% A Little Sumpin' Extra!~Ale
% Odell Cutthroat Porter
% Miner's Daughter Oatmeal Stout
% Rare Bourbon County Stout
%
% Founders KBS (Kentucky Breakfast Stout)
% Samuel Smith's Nut Brown Ale
% Salvator Doppel Bock
% Oil Of Aphrodite - Rum Barrel Aged
% Great Lakes Grassroots Ale
% Blue Dot Double India Pale Ale
% Calistoga Wheat
% Dogwood Decadent Ale
% Traquair Jacobite
% Cantillon Gueuze 100\% Lambic
%
%
\end{tabularx}
\end{minipage}
\end{table}
%\todo{Table caption above or below? I think usually above but check}


%\section{Case Studies on Recommendation Diversity}
\section{Case Studies on Recommendation and Consumption Diversity}
%\todo{This isn't really what the section is about, but maybe okay}

In \Cref{sec:?}, we saw an empirical study of diversity on \emph{YouTube}, which argued that recommender systems led to diverse views, though this analysis was limited in that the point of comparison was a popularity-based alternative (which is almost certainly not diversity-inducing). Below we explore a few additional case-studies that study diversity within the context of music (\Cref{sec:spotify}) and news (\Cref{sec:?}) recommendation, attempting to characterize users in terms of their consumption patterns, and potentially how to guide users to more diverse content.

\subsection{Diversity on Spotify}
\label{sec:spotify}

%In \Cref{sec:?}, we saw an empirical study of diversity on \emph{YouTube}, which argued that recommender systems led to diverse views, though this analysis was limited in that the point of comparison was a popularity-based alternative (which is almost certainly not diversity-inducing).

\cite{anderson2020algorithmic} tries to study the effect that recommendation algorithms have on diversity,
%more directly.
and more critically tries to understand how different types of users respond to diverse recommendations.
%\todo{bad intro}

The paper considers the listening patterns of around 100 million users on \emph{Spotify}. Unlike (e.g.)~\cite{fleder2009blockbuster}, which defined `diversity' in terms of the Gini coefficient (i.e.,~the statistical dispersion of \emph{which} items get consumed), \cite{anderson2020algorithmic} defines diversity in terms of song representations, i.e.,~essentially the $\gamma_i$ values in a recommender system.\footnote{Of course, this version of `diversity' has its own limitations, as it assumes that the learned latent space accurately captures the diversity among items.
%\todo{Is this limitation addressed in the paper?}
}

The specific embeddings $\gamma_i$ on \emph{Spotify} are estimated using an \emph{item2vec}-like method (as we saw in \Cref{sec:item2Vec}). The \emph{musical diversity} of a user $u$'s listening activity is defined in terms of a score which they term \emph{generalist-specialist} (or $\mathit{GS}$), following previous work \cite{waller2019generalists}. Specifically, we start by defining the centroid of a user's listening history (which we'll term $\gamma_u$) as
\begin{equation}
    \gamma_u = \frac{1}{|H|} \sum_{j = 1}^{|H|} \gamma_{H_j},
\end{equation}
where $H$ is a list of songs in the user's \emph{listening history}, with repetition, such that repeated listens will count more to the average. Then the $\mathit{GS}$-score is defined as the average cosine similarity between the user representation $\gamma_u$ and the items they listen to:
\begin{equation}
    \mathit{GS}(u) = \frac{1}{|H|} \sum_{j = 1}^{|H|} \frac{\gamma_{H_j}\cdot \gamma_u}{\| \gamma_{H_j} \| \| \gamma_u \|}.
\end{equation}
Intuitively, \emph{specialists} (high $\mathit{GS}(u)$) tend to have songs $\gamma_i$ in their listening history oriented primarily in a certain direction;  \emph{generalists} (low $\mathit{GS}(u)$) do not, ostensibly corresponding to a broader range of prefrences.
%has a broad range of preferences, such that songs $\gamma_i$ in their history do not tend to `point' in any specific direction; 

Part of the analysis in \cite{anderson2020algorithmic} is a study of the relationship between diversity (as measured by $\mathit{GS}(u)$) and various other attributes. For example, less active users tend to be specialists (high $\mathit{GS}(u)$), generalist users are less likely to abandon the system (`churn'), and more likely to subscribe to subscribe to the `premium' version of the product.

However the main feature in the analysis is 
%the effect that \emph{recommendations} have on diversity. 
to study the relationship between recommendations and diversity, and in particular how generalists and specialists respond differently to algorithmic recommendations.
%\todo{Finish}
This is measured experimentally by exposing real users of \emph{Spotify} to different recommendation conditions. Three types of recommender system are used: one which merely ranks songs (within a specific predefined subgenre) by popularity; one which is a simple relevance ranker based on user-to-item similarity (essentially a form of heuristic recommendation); and one which is a learned recommender specifically trained to maximize the probability that a user will listen to a song to completion.

First, compared to popularity, recommendation approaches lead to a substantial increase in the number of songs streamed for both groups (they also lead to an increase in the number of songs \emph{skipped}, but this is more than made up for in additional streams). That is, users could be said to be more \emph{engaged} when interacting with recommendations compared to a popularity baseline.
Second, the benefit of recommendations appears across both groups (generalists and specialists), though is significantly more pronounced for specialists: this aligns with the paper's hypothesis in the sense that specialists are more sensitive to songs matching their personal relevance criteria.
%
Finally, the learned ranker confers a slight additional benefit over the relevance ranker, though the benefit is surprisingly modest, indicating that simple relevance ranking is sufficient in this context.

\paragraph{Guiding Users to More Diverse Content}
%
A follow-up paper, \cite{hansen2021shifting}, 
%https://dl.acm.org/doi/pdf/10.1145/3437963.3441775
also considers consumption patterns on \emph{Spotify}, and broadly explores the trade-offs involved in terms of algorithmic choices, diversity methods, and user satisfaction. They note (as we've seen throughout this chapter) that several ranking approaches bias recommendations toward highly-popular content that closely resembles interactions from users' histories; like \cite{anderson2020algorithmic} they also find evidence that users can in many cases be satisfied by recommendations that are more diverse and less popular. 

Several diversification techniques are explored, each of which essentially attempts to trade-off a relevance versus a diversity term. \cite{hansen2021shifting} explores the merits of each; they broadly favor reinforcement learning-based approaches as a means of swaying users toward diverse content, but note the difficulties involved in productionalizing such systems.

%\done{could say more but is okay}

\subsection{Filter Bubbles and Online News Consumption}

Much of the discussion of `filter bubbles' has been in the context of online news, where concerns generally center around whether recommender systems (or more simply, algorithmic ranking techniques) will limit the ideological diversity of content users consume.

\cite{bakshy2015exposure} studied the extent to which users on \emph{Facebook} tend to consume news that confirms to their political ideology. The analysis begins by training a supervised learning system to label news articles as `liberal,' `conservative,' or `neutral,' based on shares by users who volunteer their political affiliation as part of their profile.
%Exposure to ideologically diverse news and opinion on Facebook (or somewhere else)

The main questions of interest center around the extent to which users are exposed to (or choose to interact with) content that is aligned with their own ideology versus content which is `cross-cutting.' `Exposure' refers to algorithmic feed ranking choosing to surface the content, whereas `interaction' refers to a user's choice to click on exposed content.

There are many confounding factors in such an analysis, which the study attempts to control for. For example, users' social networks are primarily composed of friends who share a common ideology, so naturally the content users could potentially be exposed to via their social network is predominantly not cross-cutting. Likewise, users' tendency to interact with (i.e.,~click on) content is confounded by the fact that the feed ranker already factors in click probability when determining which (and how prominently) content is surfaced to the user in the first place.

After attempting to control for these effects, the study's main findings are that algorithmic ranking indeed exposes users to less ideologically diverse news than would be expected by the ideological makeup of their social group, however users interact with ideologically diverse content even less then they are exposed to it; from this, the authors conclude that individual choice plays the largest role in users' exposure to content that is ideologically homogeneous.

However, the argument above does not refute the possibility of a `filter bubble' of online news consumption, it merely argues that its primary cause (in the case of \emph{Facebook}) is not necessarily algorithmic.

\cite{flaxman2016filter}
%Filter bubbles, echo chambers, and online news consumption
%\todo_{Write this section, see if this fits in between}

\paragraph{Filter Bubbles on Google News}

%\cite{haim2018burst}
%Burst of the Filter Bubble?
%Effects of personalization on the diversity of Google News
\cite{haim2018burst} conducted an exploratory study of recommendation on \emph{Google News}, in order to determine the effects of personalization on content diversity. Like \cite{bakshy2015exposure}, they broadly argue that the effects of filter bubbles are somewhat overstated, or otherwise that the patterns of bias in recommendations are not the same as what is anecdotally understood to be a `filter bubble.'

They conduct two studies, to look at `explicit' and `implicit' personalization. Both are based on empirical observation of the actual news recommendations provided by \emph{Google News}, sampled from several synthetic user accounts. Recommendations are then compared against `traditional' (i.e.,~non-personalized, curated) news sources in terms of topic and content diversity.

In the `explicit' setting, they make use of a \emph{Google News} feature that allows users to specify the types of news they are interested in, among a set of broad categories (e.g.~sports, entertainment, politics). Annotators then labeled recommended articles according to these categories in order to quantify the alignment between the explicit preferences and the recommended articles.

The first finding is simply that \emph{Google News} does indeed respect users' explicit preferences, in the sense that the proportion of recommended articles matching the desired topic far exceeds their proportion in a non-personalized setting.

\cite{haim2018burst} also evaluates recommendations in terms of source diversity (i.e.,~in terms of the original news sources that \emph{Google News} aggregates). Here, they find surprisingly that a few somewhat niche news sources dominate recommendations, whereas more mainstream sources are underrepresented; this result is relatively consistent across each of the personalized accounts.

In the `implicit' setting, \cite{haim2018burst} made use of several social media accounts, corresponding to users with specified (but synthetic) demographics and preferences (such as a marketing manager, an elderly conservative, etc.). Each of these simulated agents then interacts with social media (liking articles on \emph{Facebook}, \emph{Google+}, etc.), after which their \emph{Google News} recommendations are compared.

The main conclusion of this second study is simply that implicit personalization has little effect on the recommended results (though there is evidence that some results are indeed personalized).

Ultimately while both \cite{bakshy2015exposure} and \cite{haim2018burst} argue against a `filter bubble' as such, both point to potential issues of bias in recommendations; \cite{bakshy2015exposure} suggest that recommendations do indeed present an overall more biased perspective compared to users' broader social groups, while \cite{haim2018burst} shows that certain niche sources tend to be over-represented in news recommendations.



% Blockbuster culture’s next rise or fall:
% The impact of recommender systems on sales diversity (already covered)


Breaking the filter bubble: democracy and design
(maybe)

% Cover:
% Exploring the filter bubble: the effect of using recommender systems on content diversity \cite{nguyen2014exploring}



\section{Fairness}
\label{sec:fairness}

%Paper: Beyond Parity: Fairness Objectives for Collaborative Filtering (http://papers.nips.cc/paper/6885-beyond-parity-fairness-objectives-for-collaborative-filtering.pdf)

\emph{Fairness} in machine learning is often defined in terms of predictions and protected characteristics. For example, when building a classifier to aid in hiring decisions, we might be interested in ensuring that men and women are ranked as `qualified' at approximately the same rate. Or, a system for predicting recidivism should not be biased against individuals of a certain race \cite{chouldechova2017fair}. 

Typically, we might 
%want the predictions of 
desire that the outputs of
our classifier $f(x_i)$ 
%to be independent of a 
do not depend on some
protected feature $x_{i,f}$ (indicating race, gender, etc.).
%
Some common definitions include, for example, \emph{demographic parity}, which states that the probability of a positive prediction (e.g.~being ranked as `qualified') should be the same whether one has the protected feature or not:
\begin{equation}
    P(f(x_i) = 1 | x_{i,f} = 1) = P(f(x_i) = 1 | x_{i,f} = 0).
    %\todo{Check definition against some paper}
\end{equation}
The related notion of \emph{equal opportunity} allows for the possiiblity that the target variable depends on the protected feature,
%\emph{among qualified candidates} (i.e.,~$y_i = 1$)
and states that \emph{among the qualified $(y_i = 1)$ or unqualified $(y_i = 0)$
%\todo_{y_i = 0 or y_i = 1}
individuals}, the probability of a positive prediction should be the same whether or not one has the protected features:
%Equal opportunity:
%\done{descriptions of both. Seems adequate}
\begin{equation}
    P(f(x_i) = 1 | x_{i,f} = 1, y_i = y) = P(f(x_i) = 1 | x_{i,f} = 0, y_i = y).
\end{equation}
These are just two examples out of dozens of possible notions of fairness that may be of interest, see e.g.~\cite{mehrabi2019survey} for a comprehensive survey.

A classifier may violate the above rules for a variety of reasons. For example, the training data may exhibit historical bias against a certain group; or, a classifier trained on highly imbalanced data may simply make inaccurate (or imbalanced) predictions for groups that are poorly represented in the data. Several machine learning techniques have been proposed to mitigate unfairness in such scenarios, for example by pre-processing the biased data \cite{kamiran2009classifying}, or by altering the classifier itself \cite{zafar2017fairness}.
%, or post-process the outputs \cite{me}.

%\cite{pedreshi2008discrimination}

In the contexts of recommendations and personalized predictions, one may have slightly different definitions and goals in terms of building a `fair' model. \cite{yao2017beyond} attempts to adapt notions of fairness to personalized recommendation contexts. The paper considers a running example of recommendation in the context of education, where course evaluations in Computer Science (for example) may primarily represent the preferences of the predominantly male population; models trained on such data (or even simple statistics compiled from the data).

\cite{yao2017beyond} introduce several metrics of fairness with respect to the outputs of a recommender system, and show that these metrics can be straightforwardly incorporated into the training objective. Their metrics are defined by dividing users into groups $g_u$ and items into `genres' or attributes $h_i$; user groups are assumed to be binary, though in the studied case of gender $g_u$ can simply be divided into the over-represented group (male) and the under-represented group (non-male).
%\todo{should specify that g is male, not g is not male}

% Briefly, the fairness metrics they define include:
% \begin{description}
  \emph{Value Unfairness} measures the extent to which one group tends to have their ratings over- or under-predicted compared to the other:
  \begin{equation}
      U_{\text{val}} = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \Bigl( \underbrace{\mathbb{E}_g[y]_i}_{\hspace{-10mm}\mathrlap{\text{expected \emph{prediction} for group $g$ on item $i$}}} - \overbrace{\mathbb{E}_g[r]_i}^{\hspace{-10mm}\mathrlap{\text{average \emph{rating} for group $g$ on item $i$}}} \Bigr) - \Bigl( \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i\Bigr) \Bigr|.
      \label{eq:valueUnfairness}
  \end{equation}
  Note that since both sides take expectations (or averages), the measure is invariant to size differences between the two groups.
  
  Value unfairness could occur in a latent factor model (like that of \Cref{eq:?}) if, for example, predicitons are dominated via the bias terms $\beta_i$; in a model in which one group was over-represented, the bias terms may essentially reflect the preferences only of the over-represented group.
  %\todo{Tenses in above sound funny}
% \end{description}

\emph{Absolute Unfairness} replaces the differences in expectation from \Cref{eq:valueUnfairness} with absolute values:
\begin{equation}
      U_{\text{abs}} = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \Bigl| \mathbb{E}_g[y]_i - \mathbb{E}_g[r]_i \Bigr| - \Bigl| \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i\Bigr| \Bigr|.
      \label{eq:absoluteUnfairness}
  \end{equation}
  Following this change, absolute unfairness now captures the extent to which one group has their ratings \emph{mispredicted} in an absolute sense more than the other. This essentially measures a difference in the system's \emph{utility} between the two groups, in the sense that if one group routinely receives recommendations with high error then the system is unlikely to be useful to them.
  %\todo{poorly phrased}
  
  Next, \cite{yao2017beyond} define \emph{under-} and \emph{over-estimation} unfairness to assess the model's tendency of the model to either under or overpredict the true ratings:
  \begin{eqnarray}
      U_{\text{under}} & = & \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \max\lbrace 0,  \mathbb{E}_g[r]_i - \mathbb{E}_g[y]_i \rbrace - \max\lbrace 0, \mathbb{E}_{\neg g}[r]_i - \mathbb{E}_{\neg g}[y]_i \rbrace \Bigr|,\\
      U_{\text{over}} & = & \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \max\lbrace 0,  \mathbb{E}_g[y]_i - \mathbb{E}_g[r]_i \rbrace - \max\lbrace 0, \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i \rbrace \Bigr|.
      \label{eq:overunder}
\end{eqnarray}
%\todo{Check 0s in equations involving max (Rebecca Hu sent me note about error)}
%\todo_{doesn't render right, comma is blocked}
These definitions are somewhat analogous to related concepts we saw when evaluating ranking models in \Cref{sec:evaluatingClassifiers}; consistently underpredicting is analogous to having low recall (failing to retrieve relevant items), whereas overpredicting is analogous to having low precision (retrieving items that are not relevant). Both can potentially reduce the utility of the recommender system for one of the groups.
%\todo{Precision and recall don't feel quite right...}

Ultimately, \cite{yao2017beyond} shows that each of the above metrics can be incorporated into recommender systems of the form given in
\Cref{eq:MFObjective}.
That is, they can be combined via a trade-off term so that the model is accurate while minimizing unfairness, e.g.:
\begin{equation}
    \frac{1}{\mathcal{T}} \sum_{(u,i) \in \mathcal{T}} \underbrace{(\alpha + \beta_i + \beta_u + \gamma_i + \gamma_u - R_{u,i})^2}_{\text{accuracy}} + \lambda \underbrace{U_{\text{abs}}}_{\mathclap{\text{(absolute) fairness}}}.
    \label{eq:fairnessTradeOff}
    %\todo{Might fix alignment of underbraces}
\end{equation}
Optimization remains straightforward, as each fairness metric is differentiable with respect to the model parameters.\footnote{Or at least, has a subderivative.} The main finding of the paper is then that fairness metrics can optimized while paying only a minimal price in terms of overall model accuracy.
%\todo_{should extend the above argument to demonstrate that these types of unfairness do actually exist in data!}

%http://www.ec.tuwien.ac.at/~dimitris/research/recsys-fairness.html



\subsection{Multisided Fairness}
\label{sec:multisided}

A separate attempt to introduce fairness metrics is described in \cite{burke2017multisided}. In comparison to the fairness metrics defined above, the main difference is to consider fairness from both the perspective of users of the system (`consumers') as well as content providers (`producers'). As a motivating example, they consider a hypothetical recommendation scenario on the microfinancing website \emph{Kiva.org}, where there may be an desire to ensure that proposals
%\todo{is that the right word?}
from different businesses receive somewhat balanced representation among recommendations. More broadly, this is an instance of recommendation in a `matchmaking' setting where both sides (in this case, users and businesses) are being matched to each other; in such cases, fairness should not be defined in terms of one `side' only, but should consider the needs of both types of stakeholders. Other examples are cited including online advertising, the sharing economy, or online dating \cite{pizzato2010recon}.

To achieve this notion of fairness, they consider fairness separately from the perspective of `consumers' and `producers,' which they term $C$- and $P$-fairness. Following these definitions, the fairness metrics we studied above
%\todo{check}
would be examples of $C$-fairness. \cite{burke2017multisided} notes that $C$- and $P$-fairness are not merely symmetric definitions, and that $P$-fairness may have requirements not encountered when studying $C$-fairness, such as in the examples above. For example, in a product recommendation setting, if we wanted to encourage sales diversity (as we considered in \Cref{sec:salesDiversityAbove}, and will revisit in \Cref{sec:salesDiversityBelow}), the producers are \emph{passive} in the sense that they are not actively seeking out recommendations in the system.

Finally, \cite{burke2017multisided} considers settings of $\mathit{CP}$-fairness, where fairness must be considered from the perspectives of both sides simultaneously. We'll revisit examples of $P$-fairness and $\mathit{CP}$-fairness as we examine case studies of gender bias in \Cref{sec:genderCaseStudies}.

%Citation: Sales diversity is useful to \cite{fleder2009blockbuster}
%Have this above somewhere too

% Multisided Fairness for Recommendation (https://arxiv.org/abs/1707.00093)

Paper: Balanced Neighborhoods for Multi-sided Fairness in Recommendation (maybe)

\subsection{Group-Aware Recommendation} 

Group-aware Recommendation?
Paper: Top-N Group Recommendations with Fairness

Fairness-Aware Group Recommendation with Pareto-Efficiency

\section{Implementing Fairness Objectives in \emph{Tensorflow}}
%\todo{section should go here?}

As we discussed, the appeal of the fairness objectives such as those we developed in \Cref{sec:?} is that they can straightforwardly be incorporated into the learning objectives of standard recommenders.
%\todo{clunky sentence}

Below we'll implement `absolute fairness' as in \Cref{sec:?}. We'll use data from beer reviews (which we'll explore further in Project~\ref{proj:fairBeer}, which includes user gender information, and in which male users are substantially over-represented.
%\todo{check reference}
First, we read in the data, recording user gender along with each interaction:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
for d in parse("beer.json.gz"):
    g = 'user/gender' in d and d['user/gender'] == 'Male'
    u = d['user/profileName']
    i = d['beer/beerId']
    r = d['review/overall']
    if not u in userIDs: userIDs[u] = len(userIDs)
    if not i in itemIDs: itemIDs[i] = len(itemIDs)
    interactions.append((g,u,i,r))
\end{lstlisting}
\end{minipage}

Next we build some utility data structures to store interactions for each item according to group membership ($G$ and $\neg G$ for males and females):
%\todo{notation}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
interactionsPerItemG = defaultdict(list)
interactionsPerItemGneg = defaultdict(list)

for g,u,i,r in interactions:
    if g: interactionsPerItemG[i].append((u,r))
    else: interactionsPerItemGneg[i].append((u,r))
\end{lstlisting}
\end{minipage}

We also store item sets for each group for sampling:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
itemsG = set(interactionsPerItemG.keys())
itemsGneg = set(interactionsPerItemGneg.keys())
itemsBoth = itemsG.intersection(itemsGneg)
\end{lstlisting}
\end{minipage}

Finally, we implement the absolute fairness objective. This implementation computes the fairness objective for a single item (i.e.,~one term in the summation in \Cref{eq:}).
%\todo{check}
During training, this objective can be called for a small sample of items (just as we compute the accuracy over a small sample as in \Cref{sec:?}), and added to the accuracy term:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def absoluteUnfairness(self, i):
    G = interactionsPerItemG[i]
    Gneg = interactionsPerItemGneg[i]
    # interactions take the form (u,r)
    rG    = tf.reduce_mean(tf.convert_to_tensor([r for _,r in G]))
    rGneg = tf.reduce_mean(tf.convert_to_tensor([r for _,r in Gneg]))
    pG    = tf.reduce_mean(self.predictSample([userIDs[u] for u,_ in G], [itemIDs[i]]*len(G)))
    pGneg = tf.reduce_mean(self.predictSample([userIDs[u] for u,_ in Gneg], [itemIDs[i]]*len(Gneg)))
    Uabs = tf.abs(tf.abs(pG - rG) - tf.abs(pGneg - rGneg))
    return self.lambFair * Uabs
\end{lstlisting}
\end{minipage}
%\todo{add some comments}

\section{Case Studies on Gender Bias in Recommendation}
\label{sec:genderCaseStudies}
%\todo{Could be a section with subsections, rather than a subsection with paragraphs}

Just as \cite{yao2017beyond} used gender imbalance as a motivating example to study fairness and bias with regard to under-represented groups in recommender systems, several studies have investigated specific scenarios where recommenders exhibit significant bias, or have reduced utility for a specific gender.

\subsection{Data Resampling and Popularity Bias}

\cite{ekstrand2018all} studies a similar problem as \cite{yao2017beyond} (\Cref{sec:fairnessRec?}),
also noting that there is a substantial \emph{utility gap} between the majority versus under-represented groups (i.e.,~closest to \emph{absolute unfairness} as in \Cref{eq:absoluteUnfairness}). Bias is reported with respect to both gender and age attributes, both of which are self-reported by uses in datasets of movies and songs (from \emph{MovieLens} \cite{harper2015movielens} and \emph{Last.FM} \cite{celma2009music}).

Unlike \cite{yao2017beyond}, which corrects for this type of bias using a joint objective that balances overall utility with unfairness (\Cref{eq:fairnessTradeOff}), \cite{ekstrand2018all} uses a \emph{data resampling} approach to correct for bias. This type of approach is borrowed from \cite{kamiran2009classifying} where it was used in the context of fair classification. The basic idea is to resample the data so as to achieve equal representation among groups; practically speaking this is fairly similar to the reweighting schemes we explored in \Cref{sec:trainingBalancedClassifier}.

\cite{ekstrand2018all} also raises the potential issue of \emph{popularity bias} in recommender systems (also discussed in \cite{bellogin2011precision}), in which algorithms that work well for popular items will generally be favored over algorithms which \emph{personalize} better, but whose performance is worse for popular items.
%\todo{Clunky phrasing above}
To address this they introduce evaluation metrics that control for the effect of popularity, so that algorithms can be compared according to their degree of personalization rather than their tendency to select popular items.
%\todo{Clunky phrasing here too}

\subsection{Bias and Author Gender in Book Recommendations}
\label{sec:genderBook}

%Paper: Exploring Author Gender in Book Rating and Recommendation 

\cite{ekstrand2018exploring} 
%\todo{Is this the same as Exploring Author Gender in Book Rating and Recommendation, Michael D. Ekstrand and Daniel Kluver? Latter only has two authors}
explores bias from the perspective of book \emph{authors}. This is somewhat analogous to the idea of $P$-fairness from \Cref{sec:multisided}, given that we are interested in how recommendations could be biased against `producers' (in this case, authors of a certain gender).

Data is collected from
\emph{BookCrossing} \cite{ziegler2005improving},
\emph{Amazon} \cite{mcauley2015image},
and \emph{GoodReads} \cite{wan2018item}. An interesting component of the study is how these datasets can be augmented to incorporate the gender of each author, which is not a feature immediately available in any of the above datasets; author gender information is compiled from external sources, which is matched to records in each of the datasets.

The main research questions in \cite{ekstrand2018exploring} start by analyzing the overall gender distribution of authors in the datasets, as compared to the gender distribution among reading histories of individual users. Beyond this, they seek to study how gender bias is `propagated' by recommendation algorithms, i.e.,~the extent to which users who exhibit a moderate tendency toward authors of a certain gender will tend to have recommendations in which that gender is more extremely over-represented. Finally, they analyze the extent to which these issues can be mitigated algorithmically.

Ultimately, the study concludes that authors in all three datasets (at least those whose identities could be resolved) are predominently male. In terms of rating histories by users, the distribution is less skewed. 
In terms of recommendation algorithms, results are quite mixed, with certain algorithms and datasets leading to more or less skewed recommendations, or otherwise recommendations that mimic users' own gender preferences.

Finally, the authors find that gender imbalance in recommendations can be mitigated easily via simple re-ranking strategies, with minimal impact on performance (the strategy they use is similar to the one of \cite{ziegler2005improving}, as we studied in \Cref{sec:?}).
This analysis bears some similarity to our study of filter bubbles (\Cref{sec:?}), or the techniques used to calibrate recommendations (in this case to match a desired gender distribution rather than a genre distribution) from \Cref{sec:?}.
%or extremification (\Cref{sec:?})


%\todo{Make sure I use this citation the first time I introduce the GoodReads Data in chapter 1}

\subsection{Gender Bias in Marketing}

\cite{wan2020addressing} investigates bias in terms of how products are \emph{marketed}. For example, a user may be more (or less) inclined to purchase a clothing item if it is modeled by somebody sharing their gender, weight, age, skin-tone, etc. In some instances, these features may be directly relevant to the suitability of the item, but in others they may not be. If users are disinclined to interact with items simply because their own identity is not represented, this reduces the utility of the system to the users, represents a missed opportunity in terms of sales, and raises broader issues of representation in marketing. `Fairness' from this perspective is 
%somewhat between $C$- and $P$-fairness
an instance of $\mathit{CP}$-fairness
from \Cref{sec:multisided}, as both producers and consumers 
%stand to be harmed.
face consequences from unfair treatment.

Like the above
%\todo{check}
study, \cite{wan2020addressing} starts by assessing the extent to which these types of bias can be found in historical purchase data. They consider two settings: clothing, using a dataset from \emph{ModCloth},
%\todo{italics}
and electronics, using data from \emph{Amazon}. On \emph{ModCloth}, they are interested in whether users are less inclined to buy items if the model has a different body type than the user (e.g.~the user is plus-size but the model is not, even though the clothing is available in plus-sizes). On \emph{Amazon}, they are interested in whether ostensibly `genderless' products have different sales patterns among male and female users.

Again, the study faces difficult issues of augmenting the data, since gender and size attributes of users are not readily available. \emph{ModCloth} specifies the size of the models in marketing images, and user sizes are inferred from their historical tendency to purchase only items of a certain size. Data augmentation is more difficult on \emph{Amazon}: gender attributes in marketing images must be inferred using computer vision techniques; gender attributes of users are inferred from their purchases in the \emph{clothing} category.\footnote{Of course, clothing purchases are a rough proxy for gender identity, and users whose purchases span both gender categories are not considered.}

Indeed, the study determines that there is significant correlation between users' attributes and their purchase patterns (e.g.~male users tend to purchase electronics items marketed by male models).
%
Of course, as with gender in book recommendations
above
%\todo{check}
(\Cref{sec:genderBook}), it is hard to disentangle `bias' or `unfairness' from users' intrinsic preferences or legitimate marketing choices (e.g.~the reason that women tend to buy women's watches may be largely practical). However the goal is to determine whether bias is \emph{amplified} by recommender systems, and whether this effect can be mitigated.

The specific question that is asked is whether recommendation \emph{errors} are correlated with market segments and marketing images. This bears passing similarity to the notion of \emph{absolute unfairness} as in \Cref{eq:absoluteUnfairness}, though that measure considers unfairness only from the perspective of the \emph{user's} identity, whereas the question asked here concerns both user and item `identity' simultaneously. Specifically, four possible types of error are investigated:
\begin{equation}
\begin{tabular}{cc}
$\text{Product Image}\left\lbrace
\begin{tabular}{c}
Female\\
Male
\end{tabular}\right.$ \hspace{-6mm} &
$\left[ \begin{tabular}{cc}
$\bar{e}_{F,F}$ & $\bar{e}_{M,F}$\\
$\bar{e}_{F,M}$ & $\bar{e}_{M,M}$
\end{tabular} \right]$\\
 & $\underbrace{\begin{tabular}{cc}
 Female & Male
 \end{tabular}}_{\text{\normalsize User Identity}}$
 \end{tabular}.
\end{equation}
Under a null model, the errors should not be correlated with market segments (this can be measured via a specific statistical test).

Finding that these errors are indeed significantly correlated with market segments, \cite{wan2020addressing} seeks to address this via a loss which balances model error and error correlation:
\begin{equation}
    \overbrace{\sum_{u,i} (f(u,i) - r_{u,i})^2}^{\text{prediction error}} + \alpha \underbrace{\mathcal{L}_{\mathit{corr}}}_{\mathclap{\text{error parity on market segments}}}.
    %\todo{make not look ugly}
\end{equation}
Again, this joint loss can be optimized much like the one in \Cref{eq:fairnessTradeOff}, satisfying the fairness objective with minimal loss in prediction accuracy.

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Comparison of personalized fairness objectives.}
\label{tab:fairObj}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{|XlX|}
\hline
\textbf{Objective} & \textbf{Reference} & \textbf{Description}\\
Value Unfairness & \cite{yao2017beyond} & Neither of two groups should have their compatibility over- or under-predicted more than the other\\
Absolute Unfairness & \cite{yao2017beyond} & Neither of two groups should have their compatibility mispredicted more than the other\\
Demographic parity among recommendations & \cite{ekstrand2018exploring} & Demographics (e.g.~author gender) should be reasonably balanced among the items being recommended\\
Marketing fairness & \cite{wan2020addressing} & Individuals underrepresented in marketing (e.g.~images) should not have reduced recommendation utility\\
%\todo{poorly worded}
\hline
\end{tabularx}
\end{minipage}
\end{table}


% \section{Bias and the ‘Long-Tail’ of Personalization}

%Paper:All The Cool Kids, How Do They Fit In?
%Moved this to fairness section



\section{Justification and Trust}

Building User Trust in Recommendations via Fairness and Explanations

Designing and evaluating explanations for recommender systems


Amazon papers:
?

Netflix:
Big \& Personal: data and models behind Netflix recommendations (talks about explainability, but not very good)


%\section{Case Studies from Industry}



%Paper: Algorithmic Effects on the Diversity of Consumption on Spotify 

%\todo{Need to organize these papers. Should there be a separate `filter bubble' section, or a separate 'diversity' section?}

\section{Exercises}
%\todo{Sometimes called problems, sometimes called exercises?}
%\todo{Remove section heading, use subsection* for project. "exercises" environment already shows up in ToC}

\begin{exercises}
\item 
%Max Marginal Relevance on some recommendation problem
In this exercise we'll explore extending a recommender system to balance relevance and diversity. Start by implementing a standard latent-factor recommender, as in \Cref{chap:recsys} (or borrowing one from a previous exercise). A good dataset for this task might be one that includes category information (such as the beer dataset from Project \ref{?}, below), or some other feature that allows you to assess recommendation diversity. In this exercise we'll implement the \emph{Maximum Marginal Relevance} (MRR) criterion as in \Cref{sec:MMR} (though using a DPP (\Cref{sec:DPP}) or another diversity criterion would be similar).
%\todo{reword a little}
We'll implement and evaluate the model in the following way:
\begin{itemize}
    \item \emph{Relevance} can be defined in the usual way, i.e.,~the inner product of user and item terms $\gamma_u \cdot \gamma_i$. \emph{Diversity} could be defined in various ways, e.g.~in terms of the difference between item features $\| \gamma_i - \gamma_j \|$,
    %\todo{How do I normally write distance functions?}
    or even in terms of some simple attribute (e.g.~if you wanted to recommend beers with different alcohol levels, you could define diversity in terms of the ABV feature).
    \item Use the MMR technique to generate recommendation lists for a sample of users. The 
\end{itemize}
%\todo{Drop the above one since there's starter code now}
\item 
%Measure whether a recommender increases or decreases diversity / concentration compared to training data.
In addition to issues of diversity as we saw in Exercise \Cref{?}, we also studied \emph{concentration} effects in \Cref{sec:concentration}, whereby a recommender system can skew the distribution of recommended items toward a smaller of items than those represented in the training data.
%\todo{poorly worded}
%\todo{finish}
\item In \Cref{sec:?} we developed various fairness objectives for personalized recommender systems; although we'll explore these objectives more in Project \ref{?}, for the moment let's consider the notion of \emph{demographic parity} as in \Cref{sec:?}. In \cite{?} the authors measured demographic parity with respect to gender (of book authors), though for the purpose of this exercise you could consider any attribute associated with the items.
%\todo{finish}
\item Some other re-ranking task
%\item The dataset of beer reviews (from e.g.~\Cref{sec:someExample}) contains a considerable level of gender imbalance among users. After training a recommender system to predict ratings on this data (following e.g.~the code from \Cref{sec:surprise}), compute the four fairness metrics from \Cref{sec:?} (i.e.,~value unfairness, absolute unfairness, under- and over-estimation unfairness), comparing male ($g$) to non-male ($\neg g$) users.
%\todo{Drop this question, since we study beer in the project}
%\item Study beer dataset (or some other dataset) with respect to gender bias. Compute the various unfairness metrics on the data.
\end{exercises}

\stepcounter{projectCounter}
\label{proj:fairBeer}
\subsection{Project \theprojectCounter: Diverse and Fair Beer Recommendations}
%\todo{Some project labels are wrong since they apply to the subsection. Fix and check}

In this project we'll consider how we can improve the outputs of the types of recommendation approach we originally developed in \Cref{chap:recsys}. We'll consider the same beer datasets we've been using in various examples throughout the book. We select this dataset for two reasons:
\begin{itemize}
    \item It contains a gender attribute (like the datasets used in \Cref{sec:genderCaseStudies}), and in particular is imbalanced with respect to this attribute (the vast majority of users are male); thus we might be concerned that recommendations will have reduced utility for the underrepresented group.
    \item It contains item metadata, such as beer styles, which can be used to measure recommendation diversity, calibration, etc.
\end{itemize}
In principle this project could be completed with any similar dataset that includes (a) an attribute of interest with respect to which we can measure bias, such gender, age, etc.;~and (b) item metadata with respect to which we can measure diversity.

We'll use this dataset to analyze diversity and fairness from the following perspectives:
\begin{enumerate}
    \item Implement a recommender system to predict ratings in the dataset, e.g.~a latent factor model as in \Cref{eq:latentFactor?}.
    \item Using the above model, compute the four fairness metrics from \Cref{sec:?} (i.e.,~value unfairness, absolute unfairness, under- and over-estimation unfairness), comparing male ($g$) to non-male ($\neg g$) users.
    \item Next, we'll assess recommended items in terms of diversity. Diversity could be measured in several ways, for example you 
    %\todo{Do I ever refer to the reader as 'you'?}
    could measure diversity with respect to the distribution of recommend items, or with respect to some attribute (e.g.~the style or brand). This could be a formal measure of dispersion such as the Gini coefficient (as in \Cref{sec:gini}), or a simple 
\end{enumerate}





\chapter{Further Reading}
\label{chap:further}

\section{Online Advertising and News Recommendation}

\subsection{What Makes Ad Recommendation Different?}

Superficially, surfacing advertisements to users seems no different from any other form of personalized recommendation. That is, we can imagine learning user `preferences' and advertisement `properties' from (e.g.)~clicked advertisements in much the same way that any other recommender system is trained.

While ad recommendation does indeed have many similarities to other forms of personalized recommendation, 
%However as we discuss in this chapter, 
there are several properties that demand different solutions compared to what we have seen so far. In particular:
\begin{itemize}
 \item Advertisers have budget constraints. In most recommendation settings we can tolerate considerable imbalance among the items that are recommended (e.g.~a highly popular movie might be recommended to a substantial fraction of all users); this is impossible when recommending ads, given that each advertiser can afford to surface only a limited number of ads (and at the same time, we want to ensure that all advertisers have some ads recommended).
 \item Likewise, each user may only be shown a limited number of ads; while this seems a common enough feature in most recommendation scenarios, it is especially apparent when surfacing ads as users are unlikely to explicitly request additional ads.
 \item Ad recommendations need to be made immediately. Again, this feature is common enough in many recommender systems, but is especially challenging given the considerations above: we cannot find a globally optimal solution that maximizes utility while satisfying advertisers' budget constraints. Instead, we must develop schemes that make local decisions in a way that approximates the globally optimal solution.
 \item Ad recommendation is highly contextual. Whereas most recommender systems rely heavily on user interaction histories, these are presumably less reliable in ad recommendation scenarios (where users' interactions with ads are extremely sparse), if an interaction history is available at all. As such, one has to rely more heavily on user \emph{context} (e.g.~a user's query to a search engine).
 \item Even if a user is responsive to a certain type of ad, there is diminishing value in repeatedly showing similar ads. Instead, we must sometimes recommend ads with low expected utility in the hope of discovering new user interests (this is the basic principle behind an explore/exploit tradeoff).
\end{itemize}

These are the problems we will investigate in order to build systems for online advertising. In Section
%\todo{Finish introduction above}

Note that in this section we will mostly ignore the question of how `compatibility' between a user and an ad is estimated---this could itself be the output of a recommender system, or could simply be a bid that an advertiser places on a user or query.

\section{Matching Problems}

First we will consider the problem of incorporating constraints into (ad) recommendation problems. We want to consider situations where each user can only be shown a fixed number of ads, and each ad can only be shown to a fixed number of users. This is already an oversimplification, as advertisers would usually have a \emph{budget}, and may (for example) bid different amounts per user or query (though we consider these issues in \Cref{sec:?}).

To begin with, we'll consider the case where each ad is shown to exactly one user, and each user is shown exactly one ad. That is, we would like to select a function $\mathit{ad}(u)$ that maps users to ads, such that
\begin{equation}
    \mathit{ad}(u) = \mathit{ad}(v) \rightarrow u = v.
\end{equation}
Alternately, we could write this as an adjacency matrix $A$ such that $A_{u,a} = 1$ if $\mathit{ad}(u) = a$. Then our constraint could be written as
\begin{equation}
    \underbrace{\forall a \sum_u A_{u,a} = 1}_{\text{each advertiser shows one ad}}; \quad \underbrace{\forall u \sum_a A_{u,a} = 1}_{\text{each user sees one ad}}.
\end{equation}
%\todo{realign braces with \clap etc.}
Then, we would like to choose the mapping that maximizes the utility between users and the ads they are shown:
\begin{equation}
    \max_A \sum_{u,a} A_{u,a} f(u,a)
    \label{eq:matchingObj}
\end{equation}
where $f(u,a)$ is a measure of the compatibiliy between a user and an ad.

This type of problem is known as a \emph{matching problem}. Conceptually, it can be viewed as matching two sets of nodes to form a bipartite graph (\Cref{fig:matching}, left), where each possible edge has an associated weight (i.e.,~the compatibility between a user and an ad). Our constraint above now says that every node should be incident on one edge.

\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label=left: 1, label={[label distance=0.7mm]90:users:}] {};
\foreach \i in {2,3,...,6}
  \node[fsnode,on chain] (f\i) [label=left: \i] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label=right: 1, label={[label distance=2mm]90:ads:}] {};
\foreach \i in {2,3,...,6}
  \node[ssnode,on chain] (s\i) [label=right: \i] {};
\end{scope}
%\todo{Hardcoded vertical spacing of users: label}

\draw (s1) -- (f2);
\draw (s2) -- (f5);
\draw (s3) -- (f1);
\draw (s4) -- (f4);
\draw (s5) -- (f6);
\draw (s6) -- (f3);
% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}
\hspace{4mm}
\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label=left: 1, label={[label distance=0.7mm]90:users:}] {};
% \foreach \i in {2,3,...,6}
%   \node[fsnode,on chain] (f\i) [label=left: \i] {};
\node[fsnode,on chain] (f2) [label=left: 2] {};
\node[fsnode,on chain] (f3) [label=left: 3] {};
\node[fsnode,on chain] (f4) [label=left: 4, fill=white] {};
\node[fsnode,on chain] (f5) [label=left: 5, fill=white] {};
\node[fsnode,on chain] (f6) [label=left: 6, fill=white] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label=right: 1, label={[label distance=2mm]90:ads:}] {};
\foreach \i in {2,3,...,6}
  \node[ssnode,on chain] (s\i) [label=right: \i] {};
\end{scope}
%\todo{Hardcoded vertical spacing of users: label}

\draw (s5) -- (f3);
\draw (s1) -- (f2);
\draw (s3) -- (f1);
\end{tikzpicture}
    \caption{Ad recommendation can be viewed as a \emph{bipartite matching problem} (left), where users are shown a fixed number of ads, and each ad is shown to a fixed number of users (in this figure each ad is shown to exactly one user, and vice versa). In an online setting (right), users arrive one at a time; we seek a solution that will be as close as possible to the solution we would have obtained in an offline setting at left.}
    \label{fig:matching}
\end{figure}

Outside of ad recommendation, this type of matching problem appears in many settings, for example it is equivalent to the US's \emph{National Residency Matching Program}, in which medical school students are matched to residency programs: each student can only be matched to a single residency, and each program has a limited number of slots; the matching should be chosen so as to optimize students' preferences for programs (and vice versa). Similar problems appear in various resource allocation settings \cite{gusfield1989stable}; the original paper proposing the solution outlined below considered a setting related to college admissions \cite{gale1962college}.
%\todo{not beautiful writing}

Although \Cref{eq:matchingObj} is a combinatorial optimization problem, it admits efficient approximation (and a polynomial exact solution). The approximation we discuss below is known as \emph{stable marriage}, which draws an analogy between bipartite matching and finding (heterosexual) marriage partners: we seek a solution which, while not necessarily optimal, is `stable,' in the sense that no unmatched pair would have an incentive to `cheat.'

For each man/woman pair $m$ and $w$, we assume a compatibility $f(m,w)$.\footnote{The original solution assumed \emph{different} compatibility functions for men toward women, and women toward men, though we discard this for the application discussed here.} `Stability' then means that there should be no pairs $(m,w)$ and $(m,w')$ such that $m$ and $w'$ are matched but $f(m,w) > f(m,w')$.
%\todo{confusing writing}

and initially assume that there are an equal number of men and women (and that everyone gets married):
%\todo{put this somewhere}

Pseudocode for the stable marriage problem is given below:
\begin{itemize}
    \item All men and all women are initially ‘free’ (i.e.,~not engaged)
    \item while there is a free man $m$, and a woman he has not proposed to
    \begin{itemize}
        \item $w = \max_w f(m,w)$
        \item if ($w$ is free):
        \begin{itemize}
            \item $m$ and $w$ become engaged (and are no longer free)
        \end{itemize}
        \item else ($w$ is engaged to $m'$):
        \begin{itemize}
            \item if $w$ prefers $m$ to $m'$ (i.e.,~$f(m,w) > f(m',w)$):
            \begin{itemize}
                \item $m$ and $w$ become engaged
                \item $m'$ becomes free
            \end{itemize}
        \end{itemize}
    \end{itemize}
\end{itemize}
%\todo{actual pseudocode environment, and above is crappy}
%\todo{algorithm is basically the wikipedia one, but where does it really come from?}

Theorems:
\begin{itemize}
    \item The algorithm eventually terminates; each step includes a new proposal, and there are only $n^2$ possible proposals. Furthermore, each iteration improves the matching score of \Cref{eq:matchingObj} (or does nothing).
    \item The solution is stable. The proof is relatively straihtforward: one assumes an unstable solution in which 
    %$m$ is married to $w'$ but would prefer to be married to $w$; 
    $m$ and $w$ would prefer to be married to each other rather than their current spouses $m'$ and $w'$;
    in this case $m$ must have proposed to $w$ first (since he always proposes to his most compatible match first); so either $w$ was already engaged to somebody preferable, or $w$ accepted and later broke off the engagement; neither of these two scenarios could have ocurred if $m'$ is less compatible.
    \item Note that a consequence of the above is the possibly surprising conclusion that every matching problem as a stable solution.
    \item The solution requires $O(n^2)$ steps, since each step includes a proposal, and there are at most $n^2$ possible proposals. In practice the running time is often substantially faster.
    %\todo{What about sorting the ranked lists for each man?}
    %\todo{Cite for O(nlogn) running time?}
    \item The solution is not guanarteed to be optimal, and the optimal solution is not guaranteed to be stable. Examples of both are given in \Cref{fig:matchingOpt}.
\end{itemize}


\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
%  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=8mm]
%\node[fsnode,on chain,draw,circle] (f1) [label=left: 1, label={[label distance=0.7mm]90:women:}] {};
\node[fsnode,on chain,draw,circle] (f1) [label=left: 1] {};
\node[fsnode,on chain,draw,circle] (f2) [label=left: 2] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=8mm]
%\node[ssnode,on chain,draw,circle] (s1) [label=right: 1, label={[label distance=2mm]90:men:}] {};
\node[ssnode,on chain,draw,circle] (s1) [label=right: 1] {};
\node[ssnode,on chain,draw,circle] (s2) [label=right: 2] {};
\end{scope}
%\todo{Hardcoded vertical spacing of men/women NOT CORRECT: label}

\node[above=of f1] (w1) {women};
\node[above=of s1] (m1) {men};

\draw[dotted,thick] (s1) edge[-] (f1);
% \node [pos=0.66,above=s1] {Transpose};
\draw[thick] (s1) edge[-] (f2);
\draw[thick] (s2) edge[-] (f1);
\draw[dotted,thick] (s2) edge[-] (f2);




% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}

    \caption{Examples of matching solutions that are stable but not optimal (left), and optimal but not stable (right).
    %\todo{figure}
    \label{fig:matchingOpt}
    }

\end{figure}



%tikz:
%https://tex.stackexchange.com/questions/15088/bipartite-graphs

\subsection{AdWords}

So far, we have discussed the issue of developing recommendation approaches that consider some notion constraints or `budgets' for the perspective of users and advertisers. While incorporating constraints is potentially useful in a variety of recommendation scenarios, our solution still doesn't fully address the setting of \emph{ad} recommendation; in particular, when recommending ads, we are unlikely to see the entire set of users (or queries) in advance, before having to select advertisements.

As such, we desire an algorithm that makes decisions (i.e.,~assigns ads to users) one at a time, while still conforming to matching (or budget) constraints, as in \Cref{fig:advertising} (right).

Formally, the above describes the distinction between an \emph{offline} and an \emph{online} algorithm, i.e.,~one which sees the entire problem in advance, versus one 

\emph{AdWords} \cite{mehta2007adwords} is a specific instance of this type of recommendation problem developed for \emph{Google's} online advertising platform.

Mostly, the setting follows the one we described above, though includes a few additional components. Specifically,
\begin{itemize}
    \item Each advertiser $a$ has a \emph{bid} $f(q,a)$ that they are willing to make for each query $q$.
    \item The bid generally refers to how much the advertiser will pay \emph{if} the ad is clicked on; this is determined by an estimated \emph{click-through-rate}, $\mathit{ctr}(q,a)$. As such the expected profit would be $f(q,a) \times \mathit{ctr}(q,a)$, which one can think of as being analogous to the compatibility from \Cref{sec:sectionAbove}.
    \item Each advertiser has a \emph{budget} $b(a)$ (e.g.~for a one-week period).
    \item As in \Cref{sec:sectionAbove}, there is a limit on the number of ads that can be returned for each query.
\end{itemize}


Of course the actual implementation of AdWords contains many features not described here, for instance Adwords
%\todo{italics}
uses a second-price auction (the winning advertiser pays the amount that the \emph{second} highest bidder bid), and advertisers don't bid on exact queries, but rather are matched using a `broad matching' criteria that can include subsets, supersets, or synonyms of keywords being bid on. We refer to \cite{rajaraman2011mining} for further description of these details, and \cite{mehta2007adwords} for a more detailed technical description.

\section{Reinforcement Learning and Bandit Algorithms}

So far, when developing matching algorithms in \Cref{sec:matching?} and AdWords in \Cref{sec:adwords},
%\todo{anything else?}
we've assumed that a compatibility function between between users and ads was available in advance, and merely considered the question of how ads should be \emph{allocated} to users based on budgets and the given compatibility.

However, ad recommendation (among other settings) could present unique challenges in terms of how we estimate compatibility in the first place.

Compared to other recommendation settings, it is unlikely that we could learn user-to-ad compatibility by collecting historical data of ads users have previously interacted with. Some potential issues include:
\begin{itemize}
    \item Interaction data is extremely sparse; users likely interact with only a tiny fraction of the advertisements that are surfaced to them, making positive feedback extremely rare.
    \item Even if a user is interested in a particular topic, there are diminishing returns in repeatedly surfacing similar ads (e.g.~a user searching for rental tuxedos likely remains interested in the topic for a short time).
    \item Users do not `seek out' ads from which we might learn their interests. Thus we must sometimes surface ads before we have collected evidence that a user is interested in them. 
    \item Recommendation is highly contextual. E.g.~an ad for rental tuxedos is only worth surfacing if a user is currently searching for a closely related topic.
\end{itemize}

%\section{Case Study?}

\section{Bandit Algorithms and Reinforcement Learning}

A Contextual-Bandit Approach to Personalized News Article Recommendation \cite{li2010contextual}


The basic setup of a \emph{contextual bandit} (following e.g.~\cite{li2010contextual}) is as follows.
%\begin{itemize}
%\item 
During each step $t$, we observe a user $u_t$ and a set of actions (or `arms') $\mathcal A_t$. The actions $a \in \mathcal A_t$ reflect, for example, the space of possible advertisements that could be surfaced to a user during a particular timestep.

%\item 
In the setting of a contextual bandit, each timestep $t$ and action $a$ is associated with a feature vector $x_{a,t}$ representing the current \emph{context}; $x_{a,t}$ is intended to capture any features associated with the user and the action at a particular point in time (which could include information about their search query, etc.).

%\item
During every timestep, an action is selected, following which a reward $r_{a,t}$ is received. This reward might measure (e.g.)~whether the user clicked on the surfaced advertisement, or the payout received from doing so. Critically, the reward is only observed \emph{for the action $a$ that was selected}, that is, we cannot observe whether the user would have clicked on an ad that \emph{wasn't} shown.

%\item
Over time, the goal is to devise a strategy for selecting actions in such a way that the total reward is maximized. 


%\end{itemize}
%\todo{itemize or not?}


\begin{equation}
    E(r_{a,t}|x_{a,t}) = x_{a,t} \cdot \theta_u
\end{equation}

\subsection{News Recommendation}

Doesn't exactly go here but might go somewhere?

Turning down the noise

News search engines and the challenge to traditional journalistic
roles



\section{User Interfaces for Recommendation}

Controlling Spotify Recommendations:
Effects of Personal Characteristics on Music Recommender User
Interfaces \cite{jin2018effects}
%file:///home/jmcauley/Downloads/UMAP2018__draft.pdf



\section{Conversation and Explanation}

\subsection{Conversation}

Improving recommender
systems with adaptive conversational strategies
%Early paper

Enhancing the diversity of conversational collaborative recommendations: a comparison

Towards Conversational Recommender Systems
%https://www.kdd.org/kdd2016/papers/files/rfp0063-christakopoulouA.pdf
%Is this the paper I'm thinking of that introduces the dataset?

A Ranking Optimization Approach to Latent Linear Critiquing in Conversational Recommender System

\subsection{Explanation}

Eliciting User Preferences for Personalized Explanations for Video Summaries

%Content-based recommendation systems, Pazzani paper
%\todo{Maybe cite somewhere (not here)}


\section{Deployment Considerations}


%Active learning in recommender systems, Recommender Systems Handbook
%\todo{Maybe name section "Interactive Recommendations" or similar}


  \backmatter
%   \appendix
% if you only have one appendix, use \oneappendix instead of \appendix
%   \include{theorem}
%   \include{root}
%   \include{appnum}
%   \endappendix
%   \addtocontents{toc}{\vspace{\baselineskip}}

% the following lines will give you references at the end of the book
  \renewcommand{\refname}{Bibliography}% if you prefer this heading
%   \bookreferences % if you already have references at the end of chapters,
                  % you will need this command to start a new \chapter* heading
  \bibliography{refs}\label{refs}
  \bibliographystyle{cambridgeauthordate}

%   \cleardoublepage

% end notes, if you have them
% \theendnotes


% glossary
%   \printglossary

% indexes

% for a single index
% \printindex

% for multiple indexes using multind.sty
%   \printindex{authors}{Author index}
%   \printindex{subject}{Subject index}

\end{document}
