% Permission is granted to copy, distribute and/or modify this document
% under the terms of the GNU Free Documentation License, Version 1.3
% or any later version published by the Free Software Foundation;
% with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
% A copy of the license is included in the section entitled "GNU
% Free Documentation License".
%
% Written (C) 2012 Sergey Lisitsyn 

\chapter{Multitask learning}

In this chapter we describe multitask learning algorithms available in the 
\shogun{} toolbox. In the toolbox we include descriptions of some multitask 
learning algorithms ported from two packages: SLEP (the Sparse LEarning 
Package) and the MALSAR (Multi-tAsk Learning via StructurAl Regularization) 
package.

\section{$L_1/L_q$-norm regularized multitask learning}

One of the simplest approaches to learn linear classification and regression models in the 
multitask environment is to come with regularization based on $L_1/L_q$ norm 
of the common $w$ hyperparameter
$$
\| w \|_{1/q} = \sum_{t=1}^{T} \| w \|_q.
$$ 
That kind of regularization in the same time pulls corresponding weights 
of hyperparameters $w_t$ to be similar and pulls non-relevant feature weights
to be zero.

\subsection{Least squares linear regression}

The algorithm learns a multitask linear least squares regression model of regression 
$$
f_t(x) = \langle w_t,x \rangle + b_t, ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_w \sum_{t=1}^{T} \sum_{i \in G_t} \left(\langle w_t,x_i \rangle + b_t - y_i\right)^2
+ \lambda \| w \|_{1/q},
$$
where $G = \{ G_1, \dots, G_T \}$ is a set of tasks' non-overlapping indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \mathbb{R}$ are labels.

The algoritm is implemented in \shogunclass{CMultitaskLeastSquaresRegression}.

\subsection{Logistic regression}

The algorithm learns a multitask linear logistic model of classification 
$$
f_t(x) = sign (\langle w_t,x \rangle + b_t), ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_w \sum_{t=1}^{T} \sum_{i \in G_t} \frac{1}{|G_t|} \log (1+\exp\left(-y_i(\langle w_t,x_i \rangle + b_t)\right)
+ \lambda \| w \|_{1/q},
$$
where $G = \{ G_1, \dots, G_T \}$ is a set of tasks' non-overlapping indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \{-1,1\}$ are labels.

The algorithm is implemented in \shogunclass{CMultitaskLogisticRegression}.

\section{Tree structured group lasso multitask learning}

In some cases relations between tasks can be described via tree structure. 

\subsection{Least squares linear regression}

The algorithm learns a multitask linear least squares regression model of regression 
$$
f_t(x) = \langle w_t,x \rangle + b_t, ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_w \sum_{t=1}^{T} \sum_{i \in G_t} \left(\langle w_t,x_i \rangle + b_t - y_i\right)^2
+ \lambda \| w \|_{1/q},
$$
where $G = \{ G_0, \dots, G_T \}$ is a set of tasks' tree indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \mathbb{R}$ are labels.

The algorithm is implemented in \shogunclass{CMultitaskLeastSquaresRegression}.

\subsection{Logistic regression}

The algorithm learns a multitask linear logistic model of classification 
$$
f_t(x) = sign (\langle w_t,x \rangle + b_t), ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_{w,c} \sum_{t=1}^{T} \sum_{i \in G_t} \frac{1}{|G_t|} \log (1+\exp\left(-y_i(\langle w_t,x_i \rangle + b_t)\right)
+ \lambda \| w \|_{1/q},
$$
where $G = \{ G_0, \dots, G_T \}$ is a set of tasks' tree indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \{-1,1\}$ are labels.

The algorithm is implemented in \shogunclass{CMultitaskLogisticRegression}.

\section{Low rank approximations}

Tasks relationship can be constrained with models based on shared low-dimensional subspace. That can be done
via solving the following optimization problem:
$$
\min_{W=\left[w_1, \dots, w_T\right]} L (W) + \lambda \mathop{\mathrm{rank}}(W),
$$
where $L$ is a pre-defined loss function. It is known that the problem is NP-hard which makes it infeasible
to solve in real applications. In practice similar problem is
$$
\min_{W=\left[w_1, \dots, w_T\right]} L (W) + \lambda \|W\|_{*},
$$
where the sum of the singular values $\|W\|_{*} = \sum_i \sigma_i(W)$ is the trace norm.

\subsection{Least squares linear regression}

The algorithm learns a multitask linear least squares regression model of regression 
$$
f_t(x) = \langle w_t,x \rangle + b_t, ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_{W=\left[w_1, \dots, w_T\right]} \sum_{t=1}^{T} \sum_{i \in G_t} \left(\langle w_t,x_i \rangle + b_t - y_i\right)^2
+ \sum_i \sigma_i(W),
$$
where $G = \{ G_1, \dots, G_T \}$ is a set of tasks' non-overlapping indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \mathbb{R}$ are labels.

\subsection{Logistic regression}

The algorithm learns a multitask linear logistic model of classification 
$$
f_t(x) = sign (\langle w_t,x \rangle + b_t), ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_{W=\left[w_1, \dots, w_T\right]} \sum_{t=1}^{T} \sum_{i \in G_t} \frac{1}{|G_t|} \log (1+\exp\left(-y_i(\langle w_t,x_i \rangle + b_t)\right)
+ \sum_i \sigma_i(W),
$$
where $G = \{ G_1, \dots, G_T \}$ is a set of tasks' non-overlapping indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \{-1,1\}$ are labels.

The algorithm is implemented in \shogunclass{CMultitaskTraceLogisticRegression}.

\section{Clustered multitask learning}

Other approach assuming tasks may exhibit $k$-cluster structure. That kind of structure
makes learned models of similar tasks (i.e. tasks of one cluster) to be closer to each 
other than to other tasks. The approach can be formalized to the following optimization
problem
$$
\min_{W=\left[w_1, \dots, w_T\right]} L (W) + \alpha (\mathop{\mathrm{tr}} W^{T} W - \mathop{\mathrm{tr}} F^{T} W^{T} W F) + \beta \mathop{\mathrm{tr}} W^{T} W,
$$
where $L$ is a pre-defined loss function. The problem can be also relaxed to be convex:
$$
\min_{W=\left[w_1, \dots, w_T\right]} L (W) + \rho_1 \eta (1+\eta) (\mathop{\mathrm{tr}} (W (\eta I + M)^{-1} W^{T})
$$
subject to $\mathop{\mathrm{tr}} M = k$, $M \preceq I$, $\eta = \frac{\rho_2}{\rho_1}$.

\subsection{Least squares linear regression}

The algorithm learns a multitask linear least squares regression model of regression 
$$
f_t(x) = \langle w_t,x \rangle + b_t, ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_{W=\left[w_1, \dots, w_T\right]} \sum_{t=1}^{T} \sum_{i \in G_t} \left(\langle w_t,x_i \rangle + b_t - y_i\right)^2
+ \rho_1 \eta (1+\eta) (\mathop{\mathrm{tr}} (W (\eta I + M)^{-1} W^{T}),
$$
subject to $\mathop{\mathrm{tr}} M = k$, $M \preceq I$, $\eta = \frac{\rho_2}{\rho_1}$; where $G = \{ G_1, \dots, G_T \}$ 
is a set of tasks' non-overlapping indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \mathbb{R}$ are labels.

\subsection{Logistic regression}

The algorithm learns a multitask linear logistic model of classification 
$$
f_t(x) = sign (\langle w_t,x \rangle + b_t), ~~ t = 1, \dots, T,
$$
where $T$ is a number of tasks, from the solution of the following optimization problem:
$$
\min_{W=\left[w_1, \dots, w_T\right]} \sum_{t=1}^{T} \sum_{i \in G_t} \frac{1}{|G_t|} \log (1+\exp\left(-y_i(\langle w_t,x_i \rangle + b_t)\right)
+ \rho_1 \eta (1+\eta) (\mathop{\mathrm{tr}} (W (\eta I + M)^{-1} W^{T}),
$$
subject to $\mathop{\mathrm{tr}} M = k$, $M \preceq I$, $\eta = \frac{\rho_2}{\rho_1}$; where $G = \{ G_1, \dots, G_T \}$ 
is a set of tasks' non-overlapping indices, $\forall_i x_i$ are feature 
vectors and $\forall_i y_i \in \{-1,1\}$ are labels.

The algorithm is implemented in \shogunclass{CMultitaskClusteredLogisticRegression}.
