\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2010(2010), No. 07, pp. 1--19.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2010 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2010/07\hfil Orbital stability of periodic solutions]
{Orbital stability of periodic travelling waves for coupled
nonlinear Schr\"odinger equations}

\author[Ademir Pastor\hfil EJDE-2010/07\hfilneg]
{Ademir Pastor}

\address{Ademir Pastor Ferreira\newline
IMPA, Estrada Dona Castorina 110 \\
CEP 22460-320, Rio de Janeiro, RJ, Brazil}
\email{apastor@impa.br}

\thanks{Submitted July 26, 2008. Published January 13, 2010.}
\subjclass[2000]{76B25, 35Q55, 35Q51}
\keywords{Schr\"odinger equation; periodic travelling waves;
 orbital stability}

\begin{abstract}
 This article addresses orbital stability of periodic
 travelling-wave solutions for coupled nonlinear Schr\"odinger
 equations. We prove the existence of smooth curves of
 periodic travelling-wave solutions depending on the dnoidal-type
 functions. Orbital stability analysis is developed in the context
 of Hamiltonian systems. We consider both the stability problem
 by periodic perturbations which have the same fundamental period
 as the corresponding periodic wave and the stability problem by
 periodic perturbations having two or more times the minimal period
 as the corresponding periodic wave.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{remark}[theorem]{Remark}

\section{Introduction}

This paper is mainly concerned with the orbital stability of
periodic travelling-wave solutions for the following coupled
nonlinear Schr\"odinger equations
\begin{equation} \label{1.1}
\begin{gathered}
 iu_t+ru_{xx}+\Big(\eta|u|^2 +\sigma |w|^2 \Big)u  =  0     \\
 i\rho w_t+sw_{xx}+\Big(\frac{1}{\eta} |w|^2+
\sigma|u|^2 \Big)w =  0,
\end{gathered}
\end{equation}
where $u$ and $w$ are complex-valued functions of the variables
$x,t \in \mathbb{R}$, the parameters $\rho, \sigma$ and $\eta$ are
positive real constants, and $r=\pm 1$, $s=\pm 1$.

In optics,  \eqref{1.1} describes the interaction between
two waves of different frequencies $\omega_1$ and $\omega_2$ or two
waves of the same frequency $\omega$ but belonging to two different
polarizations. The parameters $\eta$ and $\rho$ produce an effective
asymmetry between the modes for the case of the interaction between
waves of the different frequencies. Here,  $\sigma$ is the parameter
of the cross-phase modulation, which may be determined in terms of
the parameters of the corresponding physical problem (see
\cite{Agrawal,Kivshar2,Menyuk}), and $r,s$ describe
the type of the group-velocity dispersion. For two waves of
different frequencies $\omega_1$ and $\omega_2$, one usually has
$\eta=\omega_1^2/\omega_2^2$ and $\sigma=2$; for two waves of
different polarizations in a birefringent optical medium one has
$\eta=1$ and $\sigma=2/3$ (see  e.g. \cite{Agrawal}).

More generally, \eqref{1.1} is a particular case of the
system
\begin{equation} \label{1.2}
\begin{gathered}
 iu_t+ru_{xx}-\alpha_1 u+\Big(\eta|u|^2
 +\sigma |w|^2 \Big)u+\beta_1\overline{u}^2w  =  0     \\
 i\rho w_t+sw_{xx}-\alpha_2 w+\Big(\frac{1}{\eta}
|w|^2 +\sigma|u|^2 \Big)w+\beta_2 u^3  =  0,
\end{gathered}
\end{equation}
which arises in many physical situations. For instance,  when
$r=s=1$, $\alpha_1=1$, $\eta=1/9$, $\sigma=2$, $\beta_1=1/3$, and
$\beta_2=1/9$, the system \eqref{1.2} reads as
\begin{equation} \label{1.3}
\begin{gathered}
 iu_t+u_{xx}-u+\Big(\frac{1}{9}|u|^2 +2 |w|^2 \Big)u
 +\frac{1}{3}\overline{u}^2w  =  0     \\
 i\rho w_t+w_{xx}-\alpha_2 w+\Big(9 |w|^2 +2|u|^2
\Big)w+\frac{1}{9}u^3  =  0,
\end{gathered}
\end{equation}
which describes the resonant interaction between a linearly
polarized beam of frequency $\omega$ and its third harmonic (see
\cite{Sammut1,Sammut}).

Here, we specialize the system \eqref{1.1} in the case where
$\rho=1$ and $r=s=1$, but we permit all values of $\eta, \sigma>0$.
Thus,  \eqref{1.1} reduces to
\begin{equation} \label{1.4}
\begin{gathered}
 iu_t+u_{xx}+\Big(\eta|u|^2 +\sigma |w|^2 \Big)u  =  0    \\
 i w_t+w_{xx}+\Big(\frac{1}{\eta} |w|^2+ \sigma|u|^2
\Big)w =  0.
\end{gathered}
\end{equation}

 From the mathematical viewpoint, \eqref{1.4} has been
studied by many authors (see e.g.
 \cite{Ambrosetti,deFigueiredo,Lin,Lin1,Sirakov},
\cite{Ohta}--\cite{Pelinovsky1}, but only in the context of
existence and stability of solitary-wave solutions. As far as we
know, no results concerning  the stability of periodic
travelling-wave solutions have been shown.

As a matter of fact, only a few papers  address orbital stability of
periodic travelling-wave solutions for Schr\"odinger-type systems.
We cite a few known. In \cite{Angulo2}, the authors considered a
system arising in nonlinear optics (in a medium with quadratic
nonlinearities, see \cite{Kivshar}) and they showed the existence
and stability/instability of periodic travelling waves depending on
the Jacobian elliptic function of the cnoidal type. In
\cite{Angulo5,Pastor}, the authors considered the system
\eqref{1.3}. The existence and stability/instability of periodic
travelling waves depending on the \emph{dnoidal} (in \cite{Angulo5})
and \emph{cnoidal} (in \cite{Pastor}) functions were shown. The
techniques  to obtain such results were the ones developed by
Grillakis, Shatah, and Strauss \cite {Grillakis4}, and Grillakis
\cite{Grillakis2}.

For the single cubic Schr\"odinger equation
\begin{equation}    \label{Schrodinger}
iu_t+u_{xx}+|u|^2u=0,
\end{equation}
Angulo \cite{Angulo1} established the existence of periodic
travelling waves based on the  dnoidal-type functions. By combining
the classical Lyapunov method and the Floquet theory associated to
the Lam\'e equation
$$
v'' + [\lambda-6k^2sn^2(x;k)]v=0,
$$
the author showed their orbital stability by periodic perturbations
which have the same fundamental period as the corresponding dnoidal
wave (note that one can also apply the  theory in
\cite{Grillakis4}), and orbital instability by periodic
perturbations with twice the fundamental period of the dnoidal wave.

As evidenced above, the abstract \emph{Stability/Instability Theorem}
in \cite{Grillakis4} can be applied for many dispersive equations.
However, the main difficulty when one works with travelling waves for
coupled systems, instead of one single equation, is that the
spectral analysis for the ``linearized Hamiltonian'' turns out to be
a more delicate matter. Indeed, in this case one needs to deal with
a matrix having Schr\"odinger-type operators as components. As a
consequence, in many examples, the Stability/Instability criterium
in \cite{Grillakis4} turns out to be insufficient  for a complete
stability/instability analysis of the travelling waves. The main
reason for this, is that in such approach one needs to know the
exact number of negative eigenvalues of the linearized Hamiltonian.

Grillakis \cite{Grillakis1,Grillakis2} obtained others special
instability theorems,  which get orbital instability from the linear
instability of the zero solution for the linearization of the system
around the orbit generated by the corresponding travelling wave. The
main advantage when one uses the Grillakis approach is that one does
not need to know the exact number of negative eigenvalues of the
linearized Hamiltonian, but only to have an estimate on a certain
bound (see Subsection \ref{subsection4.3} for the details).

Now, we turn our attention to the structure of the paper. The
periodic travelling-wave solutions we are interested in are of the
form
\begin{equation} \label{1.5}
u(x,t)=e^{i\gamma t}\phi_{\gamma}(x), \quad w(x,t)=e^{i\gamma
t}\psi_{\gamma}(x),
\end{equation}
where $\phi_\gamma, \psi_\gamma: \mathbb{R}\to \mathbb{R}$
are smooth periodic functions with the same fixed period $L>0$ and
$\gamma$ is a real parameter. Substituting \eqref{1.5} into
\eqref{1.4},  we get the following system of ordinary differential
equations
\begin{equation} \label{1.6}
\begin{gathered}
\phi_\gamma''-\gamma\phi_\gamma+\Big(\eta\phi_\gamma^2+\sigma\psi_\gamma^2 \Big)\phi_\gamma=  0  \\
\psi_\gamma''-\gamma\psi_\gamma +\Big(\frac{1}{\eta} \psi_\gamma^2
+\sigma\phi_\gamma^2 \Big)\psi_\gamma =  0.
\end{gathered}
\end{equation}

It is well known that \eqref{1.6} admits solitary-wave
solutions (for $\eta=1$) of the form
\begin{equation}\label{solitary}
\phi_\gamma(x)=\psi_\gamma(x)=
\sqrt{\frac{2\gamma}{\sigma+1}}\mathop{\rm sech}(\sqrt\gamma x),
\quad \gamma>0.
\end{equation}
In \cite{Pelinovsky}, the authors proved that the waves in
\eqref{solitary} are linearly stable for $\sigma>0$ and linearly
unstable for $-1<\sigma<0$. Moreover, by using the
concentration-compactness method, Ohta in \cite{Ohta} showed that
those waves are orbitally stable in the energy space
$H^1(\mathbb{R})\times H^1(\mathbb{R})$ for all $\sigma>-1$.

In the present paper, we consider two classes of periodic solutions.
First, we suppose $\psi_\gamma \equiv 0$. Then, we can find a smooth
curve of periodic solutions for \eqref{1.6} depending on the dnoidal
type function, namely,
\begin{equation}  \label{1.7}
\gamma \in \Big(\frac{2\pi^2}{L^2}, +\infty \Big) \mapsto
(\phi_{\gamma}, 0) \in H_{\rm per}^m([0,L])\times H_{\rm per}^n([0,L]),
\end{equation}
where
\begin{equation}   \label{1.8}
\phi_\gamma(x)=\eta_1dn
\Big(\frac{\sqrt\eta}{\sqrt2}\eta_1x;k\Big), \quad
k^2=\frac{\eta_1^2-\eta_2^2}{\eta_1^2},
\end{equation}
and $\eta_1, \eta_2$ are smooth functions depending on the parameter
$\gamma$ with $0<\eta_2<\eta_1$. Throughout the paper, we shall
refer to the solutions in \eqref{1.7} as \emph{semitrivial}
solutions.

Second, we  assume $\psi_\gamma=b\phi_\gamma$, for some real
constant $b\neq0$. Then, we can find another smooth curve of dnoidal
waves,
\begin{equation}\label{1.9}
\gamma \in \Big(\frac{2\pi^2}{L^2}, +\infty \Big) \mapsto
(\phi_\gamma, b\phi_{\gamma}) \in H_{\rm per}^m([0,L])\times
H_{\rm per}^n([0,L]),
\end{equation}
where, for $\theta=\eta+\sigma b^2$,
\begin{equation}\label{1.10}
\phi_\gamma(x)=\theta_1dn\Big( \frac{\sqrt\theta
}{\sqrt2}\theta_1x;k\Big),  \quad
k^2=\frac{\theta_1^2-\theta_2^2}{\theta_1^2},
\end{equation}
and $\theta_1, \theta_2$  depend smoothly on the parameter $\gamma$
and satisfy $0<\theta_2<\theta_1$. Throughout the paper, we shall
refer to the solutions in \eqref{1.9} as \emph{non-semitrivial}
solutions.


Concerning the orbital stability, for the semitrivial solutions in
\eqref{1.7}, we show that the orbit
$$
\mathcal{O}_{L}=\{(e^{ir}\phi_\gamma(\cdot+s),0); r,s \in \mathbb{R} \}
$$
is stable in the energy space
$H^1_{\rm per}([0,L])\times H^1_{\rm per}([0,L])$
provided that $\eta>\sigma$, and the orbit $\mathcal{O}_{2L}=
\{(e^{ir}\phi_\gamma(\cdot),0);\,\, r \in \mathbb{R} \}$ is
unstable in the space $H^1_{\rm per}([0,2L])\times H^1_{\rm per}([0,2L])$.
On the other hand, for the non-semitrivial solutions \eqref{1.9}
(with $\eta=b=1)$, we show that the orbit $\widetilde{\mathcal{O}}=
\{(e^{ir}\phi_\gamma(\cdot),e^{ir}\phi_\gamma(\cdot));\,\, r \in
\mathbb{R} \}$ is spectrally stable with respect to periodic
perturbations having the same fundamental period of $\phi_\gamma$,
and it is orbitally unstable in the space $H^1_{\rm per}([0,2L])\times
H^1_{\rm per}([0,2L])$. Moreover, if we assume $-1<\sigma<0$, then
$\widetilde{\mathcal{O}}$ is orbitally unstable in the space
$H^1_{\rm per}([0,L])\times H^1_{\rm per}([0,L])$. To obtain the orbital
stability result, we use the Grillakis, Shatah, and Strauss
\cite{Grillakis4} theory. However, to get the instability results,
we employ the theory developed by Grillakis
\cite{Grillakis1,Grillakis2} (see also \cite{Pelinovsky1,Shatah}).

Concerning the local well-posedness of the system \eqref{1.4}, by
introducing the Bourgain $X_{s,b}$-spaces and making use of the
contraction principle, we can prove that the system \eqref{1.4} is
locally well-posed in the spaces $H^s_{\rm per}([0,L]) \times
H^s_{\rm per}([0,L])$, $s\geq 0$. Moreover, from the conserved quantity
\begin{equation}\label{F}
\mathcal{F}(t):=\frac{1}{2}\int (|u|^2+ |w|^2)dx=\mathcal{F}(0)
\end{equation}
the local solution can be extended for any interval of time.

The paper is organized as follows: in Section \ref{wp}, we review
the results concerning the well-posedness theory. In Section
\ref{existence}, we prove the existence of smooth curves of
semitrivial and non-semitrivial solutions. Section \ref{section4} is
devoted to the orbital stability/instability of the semitrivial
solutions, whereas in Section \ref{section5} the results for the
non-semitrivial solutions are provided.


\subsection*{Notation}
For  $s\in \mathbb{R}$, the
Sobolev space $H^s_{\rm per}:=H^s_{\rm per}([0,L])$ consists of all
$L$-periodic distributions $f$ such that
$$
\|f\|_{H^s_{\rm per}}^2:=L
\sum_{k=-\infty}^{\infty}(1+k^2)^s|\widehat{f}(k)|^2 <\infty.
$$
The symbols $sn(\cdot;k)$, $dn(\cdot;k)$, and $cn(\cdot;k)$ will
denote, respectively, the Jacobian elliptic functions of snoidal,
dnoidal, and cnoidal type.

\section{Well-Posedness Theory}   \label{wp}

In this section, we review the well-posedness  theory for the system
\eqref{1.4}. For additional  details we refer the reader to
\cite{Angulo5}, where a more general result can be found. We
consider the Cauchy problem
\begin{equation} \label{2.1}
\begin{gathered}
 iu_t+u_{xx}+\Big(\eta|u|^2 +\sigma |w|^2 \Big)u  =  0, \quad x\in [0,L],
\; t \in \mathbb{R}, \\
 i w_t+w_{xx}+\Big(\frac{1}{\eta} |w|^2+ \sigma|u|^2
\Big)w = 0, \quad x\in [0,L],
\; t \in \mathbb{R}, \\
u(x,0)=u_0(x), \quad w(x,0)=w_0(x).
\end{gathered}
\end{equation}


Let $\{U(t)\}_{t=-\infty}^\infty$ be the unitary group corresponding
to the linear Schr\"odinger equation.


\begin{definition}   \label{definition2.1} \rm
Let $\mathcal{V}$ be the space of functions $f=f(x,t)$ such that
\begin{itemize}
\item[(i)] $f:[0,L]\times \mathbb{R}\to
\mathbb{C}$,
\item[(ii)] $f(x, \cdot) \in \mathcal{S}(\mathbb{R})$ (Schwartz
space) for each $ x \in [0,L]$,
\item[(iii)] $f(\cdot,t) \in C^{\infty}_{\rm per}([0,L])$
for each $ t \in \mathbb{R}$.
\end{itemize}
For $s,b \in \mathbb{R}$, we define the Bourgain space $X_{s,b}$ to
be the completion of $\mathcal{V}$ with respect to the norm
\[
 \| f\|_{X_{s,b}}  =
\Big( \sum_{n \in \mathbb{Z}} \int_{-\infty}^{\infty} \langle
n\rangle^{2s}\langle\tau+n^2\rangle^{2b} |\widehat{f}(n,\tau)|^2
d\tau \Big)^{1/2},
\]
where $\langle \cdot \rangle = 1+|\cdot|$.
\end{definition}


\begin{remark} \label{rmk2.2} \rm
 Notice that since
$ \| f\|_{X_{s,b}} = \|U(-t)f \|_{H^b(\mathbb{R}_t; H^s_{\rm per})} $,
it follows   from Sobolev's Lemma that if $b>1/2$ then $ X_{s,b}
\hookrightarrow C(\mathbb{R}_t; H^s_{\rm per}) $.
\end{remark}

Let $\zeta \in C_0^\infty(\mathbb{R})$ be a cut off function such
that $\mathop{\rm supp} \zeta \subset (-2,2)$ and $\zeta \equiv 1$ on the
interval $[-1,1]$. For each $T>0$, we define
$\zeta_T(t)=\zeta(t/T)$.

\begin{lemma}    \label{lemma2.1}
Let $s\in \mathbb{R}$, $b \in (1/2,1)$ and $T \in (0,1]$. Then,
\begin{enumerate}
    \item[(i)]  ${ \| \zeta_T U(t)v
\|_{X_{s,b}} \leq c \|v \|_{H_{\rm per}^s}}$,

\item[(ii)] $  \big\| \zeta_T \int_0^t U(t-t')f(t')dt' \big\|_{X_{s,b}}
\leq c     T ^\gamma \|f\|_{X_{s,b-1}} $,
where $\gamma$ is a positive constant.
\end{enumerate}
\end{lemma}

For a proof of the above lemma, see for example
 Kenig, Ponce, and Vega \cite{Kenig2, Kenig3}.
Next, we have a trilinear  estimate, which may be proved following
similar arguments as the ones in Bourgain \cite{Bourgain};
see also \cite{Angulo5,Bourgain1}.


\begin{lemma}   \label{lemma2.2}
Let $s \geq 0$ and $b \in (3/8, 5/8)$. Then,
\[
\|u^{\alpha_1}\overline{u}^{\alpha_2} w^{\alpha_3}
\overline{w}^{\alpha_4} \|_{X_{s,b-1}} \leq c
\|u\|_{X_{s,b}}^{\alpha_1+\alpha_2}
\|w\|_{X_{s,b}}^{\alpha_3+\alpha_4},
\]
 where $\alpha_1,\alpha_2,\alpha_3,\alpha_4 \in \{0,1,2,3 \}$ with
$\alpha_1+\alpha_2+\alpha_3+\alpha_4=3$.
\end{lemma}

With the above lemmas in hand, we are able to prove our local
well-posedness result.

\begin{theorem}[Local well-posedness]   \label{theorem2.3}
Let $s\geq 0$ and $b \in (1/2,5/8)$. For any $(u_0,w_0) \in H^s_{\rm
per}([0,L]) \times H^s_{\rm per}([0,L])$, there exist
$T=T(\|(u_0,w_0) \|_{H^s_{\rm per} \times H^s_{\rm per}})>0$ and a
unique solution $(u(t),w(t))$ of the initial-value problem
\eqref{2.1} satisfying
\begin{gather*}
(u,w) \in C([-T,T]; H^s_{\rm per}([0,L]) \times H^s_{\rm per}([0,L])), \\
(\zeta_T u, \zeta_T w) \in X_{{s,b}} \times  X_{{s,b}}.
\end{gather*}
Moreover, given $T'\in (0,T)$, there exists a neighborhood
$\mathcal{W}$ of $(u_0,w_0)$ in $H^s_{\rm per} \times H^s_{\rm per}$ such
that  the map $(u_0,w_0) \mapsto (u(t),w(t))$ from $\mathcal{W}$
into  $C([-T',T']; H^s_{\rm per}\times H^s_{\rm per})$ is Lipschitz
continuous.
\end{theorem}

\begin{proof}[Sketch of the proof]
We define the metric space of functions
$$
\mathcal{X}_M =\{ (u,w) \in X_{{s,b}} \times  X_{{s,b}};
\|(u,w)\|_{X_{{s,b}} \times  X_{{s,b}}}:=
\|u\|_{X_{s,b}}+\|w\|_{X_{s,b}} \leq M \},
$$
and the map $\Phi=(\Phi_1,\Phi_2)$, where
\begin{equation}   \label{2.2}
\begin{gathered}
 \Phi_1(u,w)(t)= \zeta_T(t)U(t)u_0+i\zeta_T(t)
\int_0^t U(t-t')\Big(\eta|u|^2u +\sigma |w|^2u \Big)(t') dt',
 \\
 \Phi_2(u,w)(t)= \zeta_T(t)U(t)w_0+i\zeta_T(t)
\int_0^t U(t-t')\Big(\frac{1}{\eta} |w|^2w+ \sigma|u|^2w \Big)(t')
dt',
\end{gathered}
\end{equation}
where $M>0$ and $T \in (0,1]$. By choosing $M$ and $T$ suitably, and
using Lemmas \ref{lemma2.1} and \ref{lemma2.2}, we can prove that
$\Phi:\mathcal{X}_M\to \mathcal{X}_M$ is a contraction.
Hence, the contraction principle implies the existence of a unique
fixed point for the integral equations \eqref{2.2}, which solves our
problem. The rest of the proof follows standard arguments, which
will be omitted.
\end{proof}

Finally, we can establish our global well-posedness result.

\begin{theorem}[Global well-posedness]  \label{theorem2.4}
For  $s \geq 0$ and $(u_0,w_0) \in H^s_{\rm per}([0,L]) \times
H^s_{\rm per}([0,L])$, the solution $(u(t),w(t))$ given in Theorem
\ref{theorem2.3} can be extend to any interval of time.
\end{theorem}

\begin{proof}
This follows from the conserved quantity
\[
\int (|u(x,t)|^2+|w(x,t)|^2)dx = \int (|u_0(x)|^2+ |w_0(x)|^2)dx,
\]
and \emph{a priori} estimates (see e.g. \cite{Bourgain1}).
\end{proof}

\begin{remark} \label{rmk2.7} \rm
The same ideas used to prove Theorems \ref{theorem2.3}  and
\ref{theorem2.4} can be applied to show local and global
well-posedness results for the system \eqref{1.2} (see
\cite{Angulo5}).
\end{remark}

\section{Existence of smooth curves of dnoidal waves}
\label{existence}

This section is devoted to establishing the existence of smooth
curves of periodic travelling-wave solutions for the system
\eqref{1.4} having the form
\begin{equation} \label{3.1}
u(x,t)=e^{i\gamma t}\phi_{\gamma}(x), \quad
w(x,t)=e^{i\gamma t}\psi_{\gamma}(x),
\end{equation}
where $\phi_\gamma, \psi_\gamma: \mathbb{R} \to \mathbb{R}$
are smooth periodic functions with the same fixed period $L>0$ and
$\gamma$ is a real parameter. Thus,  $\psi_\gamma=\psi$ and
$\phi_\gamma=\phi$ must satisfy the system of ordinary differential
equations
\begin{equation} \label{3.2}
\begin{gathered}
\phi''-\gamma\phi+\Big(\eta\phi^2+\sigma\psi^2 \Big)\phi =  0  \\
\psi''- \gamma\psi +\Big(\frac{1}{\eta} \psi^2 +\sigma\phi^2
\Big)\psi =  0.
\end{gathered}
\end{equation}
To solve this system, we  consider $\psi=b\phi$, for
some real constant $b$, and analyze two cases.


\noindent\textbf{Case 1 (semitrivial solutions)}: $b=0$.
In this case,  \eqref{3.2} reduces to the
differential equation
\begin{equation} \label{3.3}
\phi''- \gamma\phi+\eta\phi^3=0.
\end{equation}
It is well known that this equation has a positive solution of the
form
\begin{equation} \label{3.4}
\phi(x)=\eta_1 dn \Big(\frac{\sqrt\eta}{\sqrt2}\eta_1x;k \Big),
\quad k^2=\frac{\eta_1^2-\eta_2^2}{\eta_1^2},
\end{equation}
where $\eta_1, \eta_2$ are real constants satisfying
$\eta_1>\eta_2>0$.

In the sequel, we prove that the parameters $\eta_1,\eta_2$ can be
chosen such that the function $\phi$ in \eqref{3.4} has fundamental
period $L$ (here we give only the main ingredients, for details see
\cite{Angulo1},\cite{Angulo5}). Indeed, it is easy to see that
parameters $\eta_1$, $\eta_2$ must satisfy
\begin{equation} \label{3.5}
\begin{gathered}
\eta_1^2+\eta_2^2=\frac{2\gamma}{\eta}=2\omega  \\
-\eta_1^2\eta_2^2= \frac{4}{\eta}B_\phi,
\end{gathered}
\end{equation}
where $\omega:=\gamma/\eta$ and $B_\phi$ is an integration constant.
Therefore, for a fixed $\omega>0$, we get from \eqref{3.5} that
$0<\eta_2<\sqrt\omega < \eta_1< \sqrt{2\omega}$. Since $dn$ has
fundamental period $2K(k)$, it follows that the function $\phi$ in
\eqref{3.4} has fundamental period
$$
T_{\phi}=\frac{2\sqrt{2}}{\eta_1\sqrt\eta}K(k).
$$
 From \eqref{3.5}, we can see $T_\phi$ as a function depending only
on $\eta_2$ (since $\omega>0$ is fixed), namely,
$$
T_{\phi}(\eta_2)=\frac{2\sqrt{2}}{\sqrt\eta
\sqrt{2\omega-\eta_2^2}}K(k(\eta_2)), \quad
k^2(\eta_2)=\frac{2\omega-2\eta_2^2}{2\omega-\eta_2^2}.
$$
But, since $T_{\psi}(\eta_2) \to \infty$ as $\eta_2
\to 0^+$, $T_{\psi}(\eta_2) \to \frac{\pi
\sqrt{2}}{\sqrt{\omega \eta}}$ as $\eta_2 \to
\sqrt{\omega}$, and the function $\eta_2 \in (0, \sqrt{\omega})
\mapsto T_{\phi}(\eta_2)$ is strictly decreasing, we conclude that
$T_\phi> \frac{\pi \sqrt{2}}{\sqrt{\omega \eta}}$.

Hence, given $L>0$, by fixing $\omega>0$ such that
$\sqrt\omega>\frac{\pi\sqrt2}{L\sqrt\eta}$, there is a unique
$\eta_2=\eta_2(\omega) \in (0,\sqrt\omega)$ such that the dnoidal
wave $\phi$ in \eqref{3.4} has fundamental period $L=T_\phi$.

In addition, we can construct, for each $L>0$ fixed, a smooth
curve of dnoidal-wave solutions  depending on the parameter $\omega$
(and hence on $\gamma$) such that each element of the curve has
fundamental period $L$. More precisely, we prove the following.

\begin{theorem} \label{theorem3.1}
Let $L>0$ be fixed. The following statements hold.
\begin{itemize}
\item[(i)] There is a smooth function
$\Gamma:\big(\frac{2\pi^2}{\eta L^2}, +\infty \big)
\to \mathbb{R}$ such that
$$
\frac{2\sqrt{2}}{\sqrt\eta \sqrt{2\omega-\eta_2^2}}K(k)=L,
$$
where  $\eta_2=\Gamma(\omega)$ and
$k^2=k^2(\omega)=\frac{2\omega-2\eta_2^2}{2\omega-\eta_2^2}$.

\item[(ii)] The function $\Gamma$ in \textrm{(i)} is strictly decreasing
and the modulus  $k=k(\omega)$ satisfies $\frac{dk}{d\omega}>0$.

\item[(iii)] For every $\gamma \in
\big(\frac{2\pi^2}{L^2}, +\infty \big)$ and
$\omega=\omega(\gamma)=\gamma/\eta$, the dnoidal wave defined in
\eqref{3.4},
$$
\phi_\gamma(x):=\phi_{\omega(\gamma)}(x)=\sqrt{2\omega-\eta_2^2}
\, dn \Big(\frac{\sqrt{\eta(2\omega-\eta_2^2)}}{\sqrt2}x;k
    \Big)
$$
has fundamental period $L$ and satisfies \eqref{3.3}. Moreover, the
mapping
$$
\gamma \in \big(\frac{2\pi^2}{L^2}, +\infty \big) \mapsto
\phi_{\gamma} \in H_{\rm per}^n([0,L])
$$
is a smooth function.
\end{itemize}
\end{theorem}

\begin{proof}
 The proof is an application of the Implicit
Function Theorem. We refer to \cite{Angulo1} or \cite{Angulo5} for
the details.
\end{proof}

\noindent \textbf{Case 2 (non-semitrivial solutions)}: $b\neq0$.
In this case, from \eqref{3.2}, we get the system
\begin{equation} \label{3.6}
\begin{gathered}
\phi''-\gamma\phi+\big(\eta+\sigma b^2 \big)\phi^3 =  0  \\
\phi''- \gamma\phi +\big(\frac{b^2}{\eta}+ \sigma \big)\phi^3 = 0.
\end{gathered}
\end{equation}
To solve this system, we make the following assumptions:
\begin{itemize}
  \item[(H1)] $\eta+\sigma b^2=\frac{b^2}{\eta}+\sigma$,
  \item[(H2)]$(\sigma-\eta)(\sigma-\frac{1}{\eta})>0$.
\end{itemize}

These two assumptions allows us to reduce  \eqref{3.6} to
the single  equation
\begin{equation}   \label{3.7}
\phi''-\gamma \phi+ \theta\phi^3 = 0,
\end{equation}
where $\theta=\eta+\sigma b^2>0$ is a real constant. As in Case 1,
the equation \eqref{3.7} admits a dnoidal-wave solution
\begin{equation}  \label{3.8}
\phi(x)=\theta_1 dn \Big(\frac{\theta_1\sqrt{\theta}}{\sqrt{2}}x; k
\Big), \quad
k^2=\frac{\theta_1^2-\theta_2^2}{\theta_1^2},
\end{equation}
where $\theta_1, \theta_2$ are real constants and satisfy
$\theta_1^2+\theta_2^2=2\omega$, where in this case we have written
$\omega=\gamma/\theta$.

By similar arguments as in Case 1, we may verify that for every
$L>0$ and $\omega>0$  such that $\sqrt{\omega}>\frac{\pi
\sqrt{2}}{\sqrt{\theta}L}$, there exists a unique
$\theta_2=\theta_2(\omega) \in (0,\sqrt{\omega})$ such that the
dnoidal wave $\phi=\phi(\cdot;\theta_1(\omega); \theta_2(\omega))$,
given in \eqref{3.8}, has fundamental period $L=T_{\phi}$.


\begin{remark}   \label{remark3.4} \rm
Formally, the periodic-wave solution \eqref{3.8} contains the
solitary-wave solution \eqref{solitary}. Indeed, as $\theta_2 \to
0^+$ it follows that $\theta_1 \to \sqrt{2\gamma/(\eta+\sigma b^2)}$
and $dn(\cdot;1^-)\sim \mathop{\rm sech}(\cdot)$. Thus,
$$
\phi(x) \sim \sqrt{\frac{2\gamma}{\eta+\sigma b^2}}
\mathop{\rm sech}(\sqrt\gamma x),
$$
\end{remark}

As in Theorem \ref{theorem3.1}, we can prove the following.


\begin{theorem}    \label{theorem3.2}
Let $L>0$ be fixed.  The following statements hold.
\begin{itemize}
\item[(i)] There exists a smooth function
$\Lambda:\big(\frac{2\pi^2}{\theta L^2}, +\infty \big)
\to \mathbb{R}$ such that
$$
\frac{2\sqrt{2}}{\sqrt\theta\sqrt{2\omega-\theta_2^2}}K(k)=L,
$$
where  $\theta_2=\Lambda(\omega)$ and
$k^2=k^2(\omega)=\frac{2\omega-2\theta_2^2}{2\omega-\theta_2^2}$.

\item[(ii)] The function $\Lambda$ in \textrm{(i)} is strictly decreasing
and the modulus  $k=k(\omega)$ satisfies $\frac{dk}{d\omega}>0$.

\item[(iii)] For every $\gamma \in
\big(\frac{2\pi^2}{L^2}, +\infty \big)$ and
$\omega=\omega(\gamma)=\gamma/\theta$, the dnoidal wave defined in
\eqref{3.8},
$$
\phi_\gamma(x)=\phi_{\omega(\gamma)}(x)=
\sqrt{2\omega-\theta_2^2}\, dn
\Big(\frac{\sqrt{\theta(2\omega-\theta_2^2)}}{\sqrt2}x; k \Big)
$$
has fundamental period $L$ and satisfies \eqref{3.7}. Moreover, the
mapping
$$
\gamma \in \Big(\frac{2\pi^2}{L^2}, +\infty \Big) \mapsto
\phi_{\gamma} \in H_{\rm per}^n([0,L])
$$
is a smooth function.
\end{itemize}
\end{theorem}

\section{Orbital Stability/Instability of semitrivial solutions}
\label{section4}

In this section, we prove our results concerning the orbital
stability/instability of the semitri\-vial solutions given in
Theorem \ref{theorem3.1}. First, we note that the system \eqref{1.4}
may be written as a Hamiltonian system. Indeed, by writing $u=P+iQ$,
$w=R+iS$, and $U=(P,R,Q,S)$, we rewrite \eqref{1.4} as
\begin{equation}  \label{Hamiltonian}
\frac{\partial U}{\partial t}(t)=J\mathcal{H}'(U(t)),
\end{equation}
where  $J$ is the skew-symmetric matrix
\begin{equation}   \label{4.1}
J=  \begin{pmatrix} 0 & 0 & 1 & 0    \\
0 & 0 & 0 & 1   \\
-1 & 0 & 0 & 0    \\
0 & -1 & 0 & 0
\end{pmatrix},
\end{equation}
and $\mathcal{H}$ is the energy functional
\begin{equation} \label{4.2}
\begin{aligned}
\mathcal{H}(P,R,Q,S)
&=  \frac{1}{2}\int
\Big\{(P_x^2+Q_x^2)+(R_x^2+S_x^2)-\frac{\eta}{2}(P^2+Q^2)^2{}
-\frac{1}{2\eta}(R^2+S^2)^2 {} \\
&\quad -\sigma(P^2+Q^2)(R^2+S^2) \Big\} dx.
\end{aligned}
\end{equation}
In \eqref{Hamiltonian}, $\mathcal{H}'$ denotes the Fr\'echet
derivative of $\mathcal{H}$.

Recall that by using this notation, the functional $\mathcal{F}$ in
\eqref{F} reads as
\[
\mathcal{F}(P,R,Q,S) =  \frac{1}{2}\int \big\{(P^2+Q^2)+(R^2+S^2)
\big\} dx.
\]

\subsection{Spectral Analysis}

We consider $L>0$ fixed and define $\Phi=(\phi_\gamma,0,0,0)$, where
$\phi=\phi_\gamma$ is the dnoidal wave given by Theorem
\ref{theorem3.1}. Next, we consider the linearized operator
\begin{equation} \label{4.3}
\mathcal{L_{\gamma}}=\mathcal{H}''(\Phi)+\gamma \mathcal{F}''(\Phi)=
\begin{pmatrix}
\mathcal{L}_{R} & 0     \\
0 & \mathcal{L}_{I}
\end{pmatrix},
\end{equation}
where $\mathcal{L}_{R}$ and $\mathcal{L}_{I}$ are $2\times 2$ matrix
operators defined by
\begin{equation} \label{4.4}
\mathcal{L}_{R} = \begin{pmatrix}
\mathcal{L}_1 & 0     \\
0 & \mathcal{L}_3
\end{pmatrix}, \quad
\mathcal{L}_{I} = \begin{pmatrix}
\mathcal{L}_2 & 0     \\
0 & \mathcal{L}_3
\end{pmatrix},
\end{equation}
with
\begin{equation}  \label{4.5}
\mathcal{L}_{1} = -\frac{d^2}{dx^2}+ \gamma-3\eta\phi^2, \quad
\mathcal{L}_{2} = -\frac{d^2}{dx^2}+ \gamma-\eta\phi^2, \quad
\mathcal{L}_{3} = -\frac{d^2}{dx^2}+\gamma -\sigma\phi^2.
\end{equation}
In the sequel, we study the spectrum of the diagonal operator
$\mathcal{L}_\gamma$.

\begin{theorem} \label{theorem4.1}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.1}. Consider the operator $\mathcal{L}_\gamma$ in
\eqref{4.3} defined in $L^2_{\rm per}([0,L])$ with domain
$H_{\rm per}^2([0,L])$. The following statements hold.
\begin{itemize}
\item [(i)] If $n(\mathcal{L}_\gamma)$ denotes the number of
    negative eigenvalues of $\mathcal{L}_\gamma$ (counting multiplicities), then
    $n(\mathcal{L}_\gamma)=2k+1$, for some $k \in \mathbb{N}\cup\{0\}$. Moreover, the remainder
    of the spectrum is constituted by a discrete set of
    eigenvalues.
\item [(ii)] The kernel of $\mathcal{L}_\gamma$,
    $\ker(\mathcal{L}_\gamma)$, is at least two-dimensional and
    contains the space spanned by $(\phi',0,0,0)$ and
    $(0,0,\phi,0)$.
\end{itemize}
\end{theorem}

To prove Theorem \ref{theorem4.1},  the following lemma is
fundamental.

\begin{lemma} \label{lemma4.2}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.1}. Then the following spectral properties hold.
\begin{itemize}
\item [(i)] The operator $\mathcal{L}_{1}$ in $(\ref{4.5})$ defined
    in $L^2_{\rm per}([0,L])$ with domain $H_{\rm per}^2([0,L])$ has exactly one negative
    eigenvalue which is simple;  zero is an eigenvalue which is
    simple with eigenfunction $\phi'$. Moreover, the remainder
    of the spectrum is constituted by a discrete set of
    eigenvalues.
\item [(ii)] The operator $\mathcal{L}_{2}$ in $(\ref{4.5})$
    defined in $L^2_{\rm per}([0,L])$ with domain $H_{\rm per}^2([0,L])$
    has no negative eigenvalues; zero is a simple eigenvalue
    with eigenfunction $\phi$. Moreover, the remainder
    of the spectrum is constituted by a discrete set of
    eigenvalues.
\end{itemize}
\end{lemma}

\begin{proof}
The proof is essentially the same one as in
\cite[Theorems 3.1 and 3.2]{Angulo1} and
\cite[Theorem 4.1]{Angulo5}, with obvious
modifications. Note that part (ii) is an immediate consequence of
the Floquet theory (see e.g. \cite{Magnus}), since
$\mathcal{L}_{2}\phi=0$ and $\phi$ has no zeros in $[0,L]$.
\end{proof}

\begin{proof}[Proof of Theorem \ref{theorem4.1}]
Part (ii) follows immediately from Lemma \ref{lemma4.2}. To prove
part (i), we first note that, from Lemma \ref{lemma4.2},
$\mathcal{L}_\gamma$ always has a negative eigenvalue (which comes
from the operator $\mathcal{L}_1$). Moreover, from the definition of
$\mathcal{L}_\gamma$, we see that if $\lambda$ is an eigenvalue of
$\mathcal{L}_3$ (defined in $L^2_{\rm per}([0,L])$ with domain
$H_{\rm per}^2([0,L])$) then it is a double eigenvalue of
$\mathcal{L}_\gamma$. Therefore, since $\mathcal{L}_2$ has no
negative eigenvalues, we have proved the theorem.
\end{proof}

\begin{corollary} \label{corollary4.3}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.1}. Consider the operator $\mathcal{L}_\gamma$ in
\eqref{4.3} defined in $L^2_{\rm per}([0,L])$ with domain
$H_{\rm per}^2([0,L])$ and suppose that $\eta>\sigma$. Then,
\begin{itemize}
\item [(i)]  $n(\mathcal{L}_\gamma)=1$.
\item [(ii)] $\ker(\mathcal{L}_\gamma)$ is  two-dimensional and
     spanned by $(\phi',0,0,0)$ and
    $(0,0,\phi,0)$.
\end{itemize}
\end{corollary}

\begin{proof}
Since $\eta>\sigma$, we have $\gamma-\eta\phi^2(x)<\gamma-\sigma
\phi^2(x)$ in $[0,L]$. Thus, because zero is the first eigenvalue of
$\mathcal{L}_2$, an application of the Comparison Theorem (see e.g.
\cite{Eastham}) gives us that $\mathcal{L}_3$ is a strictly positive
operator. Hence, Theorem \ref{theorem4.1} yields the desired result.
\end{proof}


\subsection*{Remarks}  % \label{remark4.4}
\begin{enumerate}
\item If $\sigma=\eta$ then $n(\mathcal{L}_\gamma)=1$ but $\ker(\mathcal{L}_\gamma)$ is
  4-dimensional.
\item If $\sigma>\eta$ then, from Lemma \ref{lemma4.2} and the
  Comparison Theorem, we obtain that $n(\mathcal{L}_\gamma)\geq 3$.
  Moreover, if $3\eta>\sigma>\eta$ then $n(\mathcal{L}_\gamma)= 3$,
  and $\ker(\mathcal{L}_\gamma)$ is
     spanned by $(\phi',0,0,0)$ and
    $(0,0,\phi,0)$.
\end{enumerate}

With the goal of proving a stability/instability result by periodic
perturbation having twice the wavelength of $\phi_\gamma$, we now
prove the following.

\begin{theorem}    \label{theorem4.4}
Let $\phi_\gamma=\phi$ be the dnoidal wave given by Theorem
$\ref{theorem3.1}$.
\begin{itemize}
\item[(i)] Consider the operator $\mathcal{L}_R$  in $(\ref{4.4})$
defined in $L^2_{\rm per}([0,2L])$ with domain $H_{\rm per}^2([0,2L])$. If
$\widetilde{n}(\mathcal{L}_R)$ denotes the number of negative
eigenvalues of $\mathcal{L}_R$ (counting multiplicities),  then
$\widetilde{n}(\mathcal{L}_R)=k_0+3$ for some $k_0\in \mathbb{N}$.
Moreover, there exist $k_0+2$ eigenvalues such that the
corresponding eigenfunctions are orthogonal to
$\ker(\mathcal{L}_I)$.

\item[(ii)] Consider the operator $\mathcal{L}_I$  in $(\ref{4.4})$
defined in $L^2_{\rm per}([0,2L])$ with domain $H_{\rm per}^2([0,2L])$. If
$\widetilde{n}(\mathcal{L}_I)$ denotes the number of negative
eigenvalues of $\mathcal{L}_I$ (counting multiplicities),  then
$\widetilde{n}(\mathcal{L}_I)=k_0$. Moreover, all the eigenfunctions
corresponding to negative eigenvalues are orthogonal to
$\ker(\mathcal{L}_R)$.
\end{itemize}
\end{theorem}

\begin{proof}
(i) Let $k_0$ be the number of negative eigenvalues of the operator
$\mathcal{L}_3$ (defined in $L^2_{\rm per}([0,2L])$ with domain
$H_{\rm per}^2([0,2L])$).   So, for the first part, it suffices to show
that  the operator $\mathcal{L}_1$ (defined in $L^2_{\rm per}([0,2L])$
with domain $H_{\rm per}^2([0,2L])$) has exactly 3 negative eigenvalues.
To do this, we consider the semi-periodic eigenvalue problem
\begin{equation} \label{4.6}
\begin{gathered}
\mathcal{L}_{1}\chi=\mu \chi  \\
\chi(0)=-\chi(L), \quad \chi'(0)=-\chi'(L).
\end{gathered}
\end{equation}
which is equivalent (under the transformation
$\Lambda(x)=\chi(\alpha x)$,
$\alpha=\frac{\sqrt2}{\eta_1\sqrt\eta}$) to the following
semi-periodic eigenvalue problem associated to the Lam\'e equation:
\begin{equation} \label{4.7}
\begin{gathered}
\Lambda''+[\widetilde{\mu} - 6k^2sn^2(x;k)]\Lambda=0  \\
\Lambda(0)=-\Lambda(2K),\quad \Lambda'(0)=-\Lambda'(2K),
\end{gathered}
\end{equation}
where
\begin{equation}   \label{4.8}
\widetilde{\mu} =\frac{2}{\eta\eta_1^2}[\mu- \gamma+3\eta\eta_1^2].
\end{equation}
Now, a straightforward calculation shows us that
$\widetilde{\mu}_0=1+k^2$ and $\widetilde{\mu}_1=1+4k^2$ are the
first two eigenvalues to \eqref{4.7} (see \cite{Ince}), which are
simple with eigenfunctions given, respectively, by
$$
\Lambda_{1,sm}(x)=cn(x;k)dn(x;k), \quad
\Lambda_{2,sm}(x)=sn(x;k)dn(x;k).
$$
Thus, from \eqref{4.8} and the Floquet theory, we obtain the first
part.

For the second part, we note that if $\mu_1, \mu_2$ denote the
corresponding negative eigenvalues for \eqref{4.6}, via \eqref{4.8},
then the unique (up to a constant) eigenfunction for $\mathcal{L}_1$
associated to $\mu_i,i=1,2$ is given by
$\chi_i(x)=\Lambda_{i,sm}(\frac{1}{\alpha}x)$, and so $(\chi_i,0)$
is an eigenfunction for $\mathcal{L}_R$ associated to $\mu_i,i=1,2$.
Next, let $(u,v)\in \ker(\mathcal{L}_I)$, with $u\neq 0$. It follows
from Lemma \ref{lemma4.2} that $u=c\phi$ for some real constant $c$.
Since
$$
\int_0^{4K} dn^2(x;k)cn(x;k)dx=\int_0^{4K} dn^2(x;k)sn(x;k)dx=0,
$$
it is easy to see, from the explicit form of $\phi$ and $\chi_i$,
that
$$
\langle (\chi_i,0), (u,v) \rangle_{L^2_{\rm per}([0,2L])}=c
\int_0^{2L}\chi_i(x) \phi(x) dx=0, \quad i=1,2.
$$
Furthermore, since $\mathcal{L}_3$ is a self-adjoint operator all
its eigenfunctions are two-to-two orthogonal, and so, all the
eigenfunctions of $\mathcal{L}_R$ corresponding to negative
eigenvalues, which come from $\mathcal{L}_3$ (if they exist) are
orthogonal to $\ker(\mathcal{L}_I)$.

(ii) Since $\mathcal{L}_2$ has no negative eigenvalues, all the
negative eigenvalues of $\mathcal{L}_I$ come from the operator
$\mathcal{L}_3$ (if there exists any negative eigenvalue).
Therefore, the statement follows because $\mathcal{L}_3$ is a
self-adjoint operator. This completes the proof of the theorem.
\end{proof}

\subsection{Orbital Stability}   \label{orbital stability}

In this subsection, we establish the stability result for the
periodic travelling waves $(e^{i\gamma t}\phi_\gamma(x),0)$, where
$\phi_\gamma$ is a dnoidal wave given by Theorem \ref{theorem3.1}.
To  make clear our notion of orbital stability, we note that the
system \eqref{1.1} has phase and translation symmetries, i.e., if
$(u(x,t),w(x,t))$ is a solution of \eqref{1.1} so are
$$
(e^{is}u(x,t),e^{is}w(x,t))  \quad \mbox{and} \quad
(u(x+r,t),w(x+r,t)),
$$
for any $r,s \in \mathbb{R}$ (we denote these symmetries by $T_p(s)$
and $T_{tr}(r)$, respectively). Therefore, by orbital stability we
mean stability modulo phase and space translation. More precisely.

\begin{definition} \label{definition5.1} \rm
Let $X_1=H_{\rm per}^1([0,L])\times H_{\rm per}^1([0,L])$.
A travelling-wave
solution for \eqref{1.1}, $\Phi(x,t)=(e^{i\gamma t}\phi_\gamma(x),
e^{i\gamma t}\psi_\gamma(x))$, is said to be orbitally stable in
$X_1$ (or $X_1$-stable) if for every $\varepsilon>0$ there exists a
$\delta>0$ such that if $z_0 \in X_1$ and $\|z_0-(\phi_\gamma,
\psi_\gamma) \|_{X_1}< \delta$, then the solution $z(t)=(u(t),w(t))$
of \eqref{1.1} with $z(0)=z_0$ exists for all $t$ and satisfies
\[
\sup_{t\in{\mathbb{R}}} \inf_{s,r \in {\mathbb{R}}} \|
z(t)-T_p(s)T_{tr}(r)(\phi_\gamma,\psi_\gamma) \|_{X_1} <
\varepsilon.
\]
Otherwise, we say that $\Phi(x,t)$ is orbitally unstable in $X_1$
(or $X_1$-unstable).
\end{definition}

Our stability result is as follows.

\begin{theorem}  \label{theorem4.6}
Let  $\gamma \in \big( \frac{2\pi^2}{L^2}, \infty \big)$, and
assume that $\sigma, \eta>0$ satisfy $\eta>\sigma$. Then, for
$\phi_{\gamma}$ given by Theorem $\ref{theorem3.1}$, the periodic
travelling waves $\Phi_\gamma(x,t)=(e^{i\gamma t}\phi_{\gamma}(x),0)$
are orbitally stable in $X_1$.
\end{theorem}

\begin{proof}
The idea is to apply the theory developed by Grillakis, Shatah, and
Strauss \cite{Grillakis4} for abstract Hamiltonian system. To do so,
we first note that from Corollary \ref{corollary4.3}, the real
Hilbert space $X_{\mathbb{R}}:=[H_{\rm per}^1([0,L])]^4$ can be
orthogonally decomposed as
\[
 X_{{\mathbb{R}}}=\mathcal{N} \oplus \ker(\mathcal{L}_\gamma) \oplus \mathcal{P},
\]
where $\mathcal{N}$ denotes the negative eigenspace of
$\mathcal{L}_{\gamma}$ and $\mathcal{P}$ is a closed subspace such
that $\langle\mathcal{L}_{\gamma}p,p \rangle \geq \vartheta_0
\|p\|_{X_1}^2$, for all $p \in \mathcal{P}$ and some
$\vartheta_0>0$.

Next, for  $\gamma \in I=\big( \frac{2\pi^2}{L^2}, \infty \big)$
and $\Phi_{\gamma}=(\phi_{\gamma},0,0,0)$, we define the real
function
\begin{equation} \label{functiond}
d(\gamma)=\mathcal{H}(\Phi_{\gamma})+ \gamma
\mathcal{F}(\Phi_{\gamma}),
\end{equation}
Hence, since $\Phi_{\gamma}$ is a critical point of the functional
$\mathcal{H}+ \gamma \mathcal{F}$, $\ker(\mathcal{L}_\gamma)$ is
two-dimensional, and $\mathcal{N}$ is one-dimensional, from the
abstract Stability Theorem in \cite{Grillakis4}, we  just need to
prove that $d''(\gamma)>0$. To this end, from \eqref{functiond}, we
immediately obtain
\begin{equation}   \label{4.9}
d'(\gamma)= \mathcal{F}(\Phi_{\gamma})=\frac{1}{2} \|\phi_{\gamma}
\|_{L_{\rm per}^2([0,L])}^2.
\end{equation}
But, from the explicit form of $\phi_{\gamma}$, we calculate
\begin{equation}   \label{4.10}
\|\phi_{\gamma} \|_{L_{\rm per}^2([0,L])}^2=\frac{8}{L}K(k)
\int_0^{K(k)} dn^2(x;k)dx=\frac{8}{L}K(k)E(k),
\end{equation}
where in the last equality we have used that $\int_0^{K(k)}
dn^2(x;k)dx=E(k)$ (see \cite[pg. 10]{Byrd}, here $E(k)$ denotes the
complete elliptic integral of the second kind). So, from \eqref{4.9}
and \eqref{4.10}, we deduce that
$$
d''(\gamma)=\frac{4}{L}\frac{d}{dk}[K(k)E(k)]\frac{dk}{d\omega}
\frac{1}{\eta}.
$$
Since the mapping $k\in (0,1) \mapsto K(k)E(k)$ is a strictly
increasing function, and because  from Theorem \ref{theorem3.1} we
have $\frac{dk}{d\omega}>0$, the theorem follows.
\end{proof}

\subsection*{Remark}
 Note that our stability result includes the
case of two waves of different polarizations and the case for two
waves of different frequencies $\omega_1$ and $\omega_2$ with
$\omega_2 \ll \omega_1$ (see the Introduction).

\subsection{Orbital Instability}  \label{subsection4.3}

In Subsection \ref{orbital stability}, we established a orbital
stability result for the periodic waves
$\Phi_\gamma(x,t)=(e^{i\gamma t}\phi_{\gamma}(x),0)$, where
$\phi_\gamma$ is given by Theorem \ref{theorem3.1}, by periodic
perturbations which have the same fundamental period of
$\phi_\gamma$. In this subsection, we ask ourselves if such waves
are stable/unstable when we consider periodic perturbations which
have twice the fundamental period of $\phi_\gamma$. As we will see
below, in this case the waves $\Phi_\gamma(x,t)$ are orbitally
unstable but in a weaker sense than in Definition
\ref{definition5.1}. Actually, here our notion of orbital stability
is slightly different from that in Subsection \ref{orbital
stability} and does not include space translations.


\begin{definition} \label{definition6.1} \rm
Let $X_2=H_{\rm per}^1([0,2L])\times H_{\rm per}^1([0,2L])$. The
orbit generated modulo phase,  $\{T_p(\gamma
s)(\phi_\gamma,\psi_\gamma); s \in {\mathbb{R}} \}$, is said to be
orbitally stable in $X_2$ (or $X_2$-stable) if for every
$\varepsilon>0$ there exists a $\delta>0$ such that if $z_0 \in X_2$
and $\|z_0-(\phi_\gamma, \psi_\gamma) \|_{X_2}< \delta$, then the
solution $z(t)=(u(t),w(t))$ of \eqref{1.1} with $z(0)=z_0$ exists
for all $t$ and sa\-tisfies
$$
{ \sup_{t\in{\mathbb{R}}} \inf_{s \in {\mathbb{R}}}
\| z(t)-T_p(s)(\phi_\gamma,\psi_\gamma) \|_{X_2} < \varepsilon.}
$$
Otherwise, the orbit is said to be orbitally unstable in $X_2$ (or
$X_2$-unstable).
\end{definition}


Here, we follow the approach introduced by Grillakis in
\cite{Grillakis1}, \cite{Grillakis2} (see also \cite{Angulo2},
\cite{Angulo5}), which get orbital instability from the linear
instability of the zero solution for the linearization of
\eqref{1.4} around the orbit $\{T_p(\gamma
s)(\phi_\gamma,\psi_\gamma); s \in {\mathbb{R}} \}$.

To start, we define the orbit $\mathcal{O}$ to be
$$
\mathcal{O}=\{T_p(\gamma s)(\phi_{\gamma},0,0,0); s \in
{\mathbb{R}} \},
$$
 where $\phi_\gamma=\phi$ is the dnoidal wave given
by Theorem \ref{theorem3.1}. Next, for $\Phi=(\phi_{\gamma},0,0,0)$
and  $U=(P,R,Q,S)$ (recall that we have written $u=P+iQ$, $w=R+iS$),
we define $V=V(t)$ to be
\[
V=T_p(-\gamma t)U-\Phi.
\]
By using the group properties of $T_p(s)$ together with the fact
that $\Phi$ is a critical point of the functional
$\mathcal{H}+\gamma \mathcal{F}$, it is easy to see, from the Taylor
expansion and \eqref{Hamiltonian}, that $V(t)$ satisfies
\begin{equation} \label{4.12}
 \frac{dV}{dt}   =  J\mathcal{L}_{\gamma}V+O(\|V\|^2),
\end{equation}
where $J$ and $\mathcal{L}_\gamma$ are the operators defined in
\eqref{4.1} and \eqref{4.3}, respectively.

To prove that the linearized equation \eqref{4.12} has zero as an
unstable solution, it is well known that it suffices to show that
$J\mathcal{L}_\gamma$ has one and finitely many eigenvalues with
strictly positive real part. Moreover, this implies that the orbit
$\mathcal{O}$ is orbitally unstable  (see \cite{Grillakis1},
\cite{Grillakis4}, \cite{Shatah}). Keeping this in mind, we prove
the following result.

\begin{theorem}  \label{theorem4.8}
Let  $\gamma \in \big( \frac{2\pi^2}{L^2}, \infty \big)$ and
$\sigma, \eta>0$. Then, for $\phi=\phi_{\gamma}$ given by Theorem
\ref{theorem3.1}, the orbit
$$
\mathcal{O}= \{T_p(\gamma t)(\phi,0,0,0); t \in {\mathbb{R}}\}
$$
is $X_2$-unstable by the periodic flow of the system $(\ref{1.4})$.
\end{theorem}

\begin{proof}
As we have already pointed out, we only have to prove that the
operator $J\mathcal{L}_{\gamma}$ has one and  finitely many
eigenvalues with strictly positive real part. But, from Lemma 5.6
and Theorem 5.8 in \cite{Grillakis4}, we know that
$J\mathcal{L}_{\gamma}$ has finitely many eigenvalues with strictly
positive real part. Now, to prove that $J\mathcal{L}_{\gamma}$ has
at least one eigenvalue with strictly positive real part, in light
of Theorem 2.6 in \cite{Grillakis2}, we define
\begin{gather*}
Y= [\ker(\mathcal{L}_{R})\cup \ker(\mathcal{L}_{I})]^{\bot},\\
\widehat{\mathcal{L}}_{R}= \mbox{restriction of }
\mathcal{L}_{R} \mbox{ on }  Y\cap H_{\rm per}^2([0,2L]) ,\\
\widehat{\mathcal{L}}_{I}^{-1}= \mbox{restriction of }
\mathcal{L}_{I}^{-1} \mbox{ on }  Y\cap H_{\rm per}^2([0,2L]).
\end{gather*}
With these notation, Theorem 2.6 in \cite{Grillakis2} states that
$J\mathcal{L}_\gamma$ has exactly
\begin{equation}   \label{4.13}
\max\{ n(\widehat{\mathcal{L}}_{R}),
n(\widehat{\mathcal{L}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1}))
\end{equation}
$\pm$ pairs of real eigenvalues, where $C(\mathcal{L})=\{ y \in Y;
\langle \mathcal{L}y, y\rangle <0 \}$ denotes the negative cone of
the operator $\mathcal{L}$ and $ d(C(\mathcal{L}))$ denotes the
dimension of a maximal linear subspace that is contained in
$C(\mathcal{L})$.


Therefore, we have proved the theorem  if we show that the number in
\eqref{4.13} is strictly positive. Since $\mathcal{L}_{R}$ is a
self-adjoint operator on $L^2_{\rm per}([0,2L])$, its negative
eigenspace is orthogonal to its kernel (the same conclusion holds
for the operator $\mathcal{L}_{I}$). Thus, from Theorem
\ref{theorem4.4}, we have $ n(\widehat{\mathcal{L}}_{R})=k_0+2$ and
$n(\widehat{\mathcal{L}}_{I})=k_0$. Moreover, from the structure of
the operators $\mathcal{L}_{R}$ and $ \mathcal{L}_{I}$, we see that
the negative cone $C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1})$ is $k_0$-dimensional; that is,
$$
d(C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1}))=k_0.
$$
Hence,
$$
\max\{ n(\widehat{\mathcal{L}}_{R}),
n(\widehat{\mathcal{L}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1}))=k_0+2-k_0=2.
$$
This completes the proof of the theorem.
\end{proof}

\section{Orbital Instability of the non-semitrivial solutions}
\label{section5}


This section is mainly devoted to prove the instability results
concerning the dnoidal-wave solutions given in Theorem
\ref{theorem3.2}. For the sake of simplicity, throughout this
section we take $\eta=1$. So, the coupled system \eqref{1.4} admits
the solutions in Theorem \ref{theorem3.2}  for $\sigma \neq 1$ and
$b^2=1$. We assume throughout that $b=1$, but a similar analysis can
be performed if $b=-1$.


\subsection{Spectral Analysis and Stability/Instability}

Fix $L>0$  and let $\widetilde{\Phi}=(\phi_\gamma,\phi_\gamma,0,0)$,
where $\phi=\phi_\gamma$ is the dnoidal wave given by Theorem
\ref{theorem3.2} (with $\eta=b=1$). Consider the linearized operator
\begin{equation} \label{5.1}
\mathcal{T_{\gamma}}=\mathcal{H}''(\widetilde{\Phi})+\gamma
\mathcal{F}''(\widetilde{\Phi})
=\begin{pmatrix}
\mathcal{T}_{R} & 0     \\
0 & \mathcal{T}_{I}
\end{pmatrix},
\end{equation}
where
\begin{gather} \label{5.2}
\mathcal{T}_{R} = \begin{pmatrix}
-\frac{d^2}{dx^2}+\gamma-\left( 3+\sigma \right) \phi_\gamma^2 &
-2\sigma\phi_\gamma^2 \\
-2\sigma\phi_\gamma^2 & -\frac{d^2}{dx^2}+\gamma-\left( 3+\sigma
\right) \phi_\gamma^2
\end{pmatrix},\\
 \label{5.3}
\mathcal{T}_{I} = \begin{pmatrix}
-\frac{d^2}{dx^2}+\gamma-\left( 1+\sigma \right) \phi_\gamma^2 & 0
\\
0 & -\frac{d^2}{dx^2}+\gamma-\left( 1+\sigma \right) \phi_\gamma^2
\end{pmatrix}.
\end{gather}
Our first result is about the study of the spectra of the
operators $\mathcal{T}_{R}$ and $\mathcal{T}_{I}$.

\begin{theorem} \label{theorem5.1}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.2} and assume that $\sigma>0$. Then the following
spectral properties hold.
\begin{itemize}
\item [(i)] If $\sigma>1$ then the operator $\mathcal{T}_{R}$ in (\ref{5.2}) defined
    in $L^2_{\rm per}([0,L])$ with domain $H_{\rm per}^2([0,L])$ has exactly one negative
    eigenvalue which is simple;  zero is a simple eigenvalue. Moreover, the remainder
    of the spectrum is constituted by a discrete set of
    eigenvalues.
\item [(ii)] If $0<\sigma<1$ then the operator $\mathcal{T}_{R}$ in (\ref{5.2})
    defined in $L^2_{\rm per}([0,L])$ with domain $H_{\rm per}^2([0,L])$
     has exactly two negative
    eigenvalue which are simple;  zero is a simple eigenvalue. Moreover the remainder
    of the spectrum is constituted by a discrete set of
    eigenvalues.
\item[(iii)] The operator $\mathcal{T}_{I}$ in (\ref{5.3})
    defined in $L_{\rm per}^2([0,L])$ with domain $H_{\rm per}^2([0,L])$ has
    no negative eigenvalues; its kernel  is two-dimensional
    and spanned by $(0,\phi)$ and $(\phi,0)$. Moreover, the remainder
    of the spectrum is constituted by a discrete set of
    eigenvalues.
\end{itemize}
\end{theorem}

\begin{proof}
Let
\begin{equation}\label{5.4}
\mathcal{T}_{1}= -\frac{d^2}{dx^2}+\gamma-3( 1+\sigma)
\phi_\gamma^2, \quad \mathcal{T}_{2}=
-\frac{d^2}{dx^2}+\gamma-\left( 1+\sigma \right) \phi_\gamma^2.
\end{equation}
It is not difficult to check that the same results as in Lemma
\ref{lemma4.2} hold for the operators $\mathcal{T}_{1}$ and
$\mathcal{T}_{2}$ replacing $\mathcal{L}_{1}$ and $\mathcal{L}_{2}$,
respectively. So, since $\mathcal{T}_{2}$ has no negative
eigenvalues and zero is a simple eigenvalue we have proved part
(iii).

To show parts (i) and (ii), we note that the operator
$\mathcal{T}_{R}$ can be diagonalized under a similarity
transformation. Indeed, let
\begin{equation}  \label{5.5}
\mathcal{A}_{R} = \begin{pmatrix}
1 & 1\\
 -\frac{1}{2} &  \frac{1}{2}
\end{pmatrix}
\end{equation}
Then
\begin{equation}  \label{5.6}
 \mathcal{T}_{DR}:=
\mathcal{A}_{R}\mathcal{T}_{R}\mathcal{A}_{R}^{-1}
= \begin{pmatrix}
\mathcal{T}_{1} & 0 \\
0 & \mathcal{T}_{3}
\end{pmatrix}
\end{equation}
where $\mathcal{T}_{1}$ is defined in \eqref{5.4} and
\begin{equation}\label{5.7}
\mathcal{T}_{3}=\mathcal{T}_{1}+4\sigma \phi_\gamma^2.
\end{equation}
Since $\mathcal{T}_{1}$ has exactly one negative eigenvalue, zero is
the second eigenvalue and $\sigma>0$, the Comparison Theorem implies
that $\mathcal{T}_{3}$ has at most one negative eigenvalue and the
second eigenvalue is strictly positive.

Now, note that from \eqref{5.7}, we can also write
\begin{equation}\label{5.8}
\mathcal{T}_{3}=\mathcal{T}_{2}+2(\sigma-1) \phi_\gamma^2.
\end{equation}
Because $\mathcal{T}_{2}$ has no negative eigenvalues, it follows
from \eqref{5.8}, the above observation,  and the Comparison Theorem
that $\mathcal{T}_{3}$ is a strictly positive operator if
$\sigma>1$, and it has a unique negative eigenvalue if $0<\sigma<1$.
This proves the theorem.
\end{proof}


\begin{remark} \label{remark5.2} \rm
It follows from Theorem \ref{theorem5.1} that the kernel of the
operator $\mathcal{T}_{\gamma}$ is 3-dimensional. Hence, the
Grillakis, Shatah, and Strauss theory cannot be applied to get a
stability result as in Theorem \ref{theorem4.6}.
\end{remark}

In view of Remark \ref{remark5.2}, we propose in proving an
instability result (modulo phase) in the same spirit of Subsection
\ref{subsection4.3}. First, we linearize \eqref{1.4} around the
orbit $\widetilde{\mathcal{O}}=\{T_p(\gamma
s)(\phi_{\gamma},\phi_\gamma,0,0); s \in {\mathbb{R}} \}$. Defining
$W=W(t)$ to be
\[
W=T_p(-\gamma t)U-(\phi_{\gamma},\phi_\gamma,0,0),
\]
we get the linearized equation
\begin{equation} \label{5.9}
 { \frac{dW}{dt}} = J\mathcal{T}_{\gamma}W,
\end{equation}
where $J$ is the matrix in \eqref{4.1} and $\mathcal{T}_\gamma$ is
defined in  \eqref{5.1}.

The first natural question is the following: is the orbit
$\widetilde{\mathcal{O}}$ unstable with regard to periodic
perturbations which have  the same fundamental period as
$\phi_\gamma$? In this case, the method of proof employed in Theorem
\ref{theorem4.8} does not give a result concerning the orbital
instability as in Subsection \ref{subsection4.3}, and we can only
prove a spectral stability result. More precisely, we prove the
following.

\begin{theorem}  \label{theorem5.3}
Let  $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.2}. Then the orbit
$$
\widetilde{\mathcal{O}}=\{T_p(\gamma
s)(\phi_{\gamma},\phi_\gamma,0,0); s \in {\mathbb{R}} \}
$$
is spectrally stable, which is to say, the spectrum of the
linearized operator $J\mathcal{T}_\gamma$, in the space
$L^2_{\rm per}([0,L])$, entirely lies on the imaginary axis.
\end{theorem}

\begin{proof}
As in the proof of Theorem \ref{theorem4.8}, we already know that
the operator $J\mathcal{T}_\gamma$ has finitely many eigenvalues
with strictly positive real part. Moreover, since
$n(\mathcal{T}_I)=0$ (see Theorem \ref{theorem5.1}), we deduce that
the unstable eigenvalues of $J\mathcal{T}_\gamma$, that is, those
with a strictly  positive real part, may occur only as real positive
eigenvalues (see e.g. \cite{Grillakis1} or \cite{Pelinovsky1}).
Thus, since $J\mathcal{T}_\gamma$ has exactly
\begin{equation}  \label{5.10}
\max\{ n(\widehat{\mathcal{T}}_{R}),
n(\widehat{\mathcal{T}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{T}}_{R}) \cap
C(\widehat{\mathcal{T}}_{I}^{-1}))
\end{equation}
$\pm$ pairs of real eigenvalues (see Subsection \ref{subsection4.3}
for the definitions), we only have to prove that such a number is
zero. But, since  $n(\mathcal{T}_I)=0$, the  number in \eqref{5.10}
reduces to $ n(\widehat{\mathcal{T}}_{R})$. So, our task is to show
that no eigenfunctions of $\mathcal{T}_{R}$, associated to negative
eigenvalues, are orthogonal to $\ker(\mathcal{T}_I)$. To do this,
let $\overrightarrow{u}=(u_1,u_2)\neq 0$ and $\lambda<0$ such that
$\mathcal{T}_{R}\overrightarrow{u}=\lambda \overrightarrow{u}$. From
\eqref{5.6}, we see that
\[
 \mathcal{A}_{R}\overrightarrow{u}
= \begin{pmatrix}
u_1+u_2 \\
-\frac{1}{2}u_1 +  \frac{1}{2}u_2
\end{pmatrix}
\]
is an eigenfunction of $\mathcal{T}_{DR}$ associated to the negative
eigenvalue $\lambda$. As a consequence, either
$\mathcal{T}_{1}u=\lambda u$ or $\mathcal{T}_{3}v=\lambda v$, where
$u=u_1+u_2$ and $v=\frac{1}{2}(u_2-u_1)$. Now, from Theorem
\ref{theorem5.1}, it follows that $u\equiv 0$ or $v\equiv 0$. This
means that either
\begin{itemize}
\item[(i)] $\overrightarrow{u}=(-u_2,u_2)$ and $\mathcal{T}_{3}u_2=\lambda
u_2$; or

\item[(ii)] $\overrightarrow{u}=(u_1,u_1)$ and $\mathcal{T}_{1}u_1=\lambda
u_1$.
\end{itemize}
Because $\mathcal{T}_{1}$ and $\mathcal{T}_{3}$ have at
most one negative eigenvalue, it follows from the Floquet theory
that if (i) occurs then $u_2>0$ and if (ii) occurs then $u_1>0$.
Hence, for any $\overrightarrow{\phi}=(\alpha_1 \phi_\gamma,
\alpha_2 \phi_\gamma) \in \ker(\mathcal{T}_{I})$ with $\alpha_1 \neq
\alpha_2$, we have $\langle \overrightarrow{u},
\overrightarrow{\phi} \rangle_{L^2_{\rm per}} \neq 0$. In consequence,
$\overrightarrow{u} \notin [\ker(\mathcal{T}_{I})]^\perp$. This
completes the proof of the theorem.
\end{proof}

In analogy to Theorem \ref{theorem4.8}, we can also prove that the
orbit $\widetilde{\mathcal{O}}$ is orbitally unstable  with regard
to periodic perturbations which have twice the fundamental period of
$\phi_\gamma$.

\begin{theorem}  \label{theorem5.4}
Let  $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
$\ref{theorem3.2}$. Then the orbit
$$
\widetilde{\mathcal{O}}=\{T_p(\gamma
s)(\phi_{\gamma},\phi_\gamma,0,0); s \in {\mathbb{R}} \}
$$
is $X_2$-unstable, in the sense of Definition \ref{definition6.1},
by the periodic flow of system $(\ref{1.4})$.
\end{theorem}

\begin{proof}
The proof follows the same arguments as in Theorem \ref{theorem4.8}.
Actually, similarly to the proof of Theorem \ref{theorem4.4}, we can
prove that the operator $\mathcal{T}_{1}$, with domain
$H^2_{\rm per}([0,2L])$ has exactly three negative eigenvalues, for
which, the eigenfunctions corresponding to the second and third
eigenvalues are orthogonal to $\phi_\gamma$. Therefore,
\[
\max\{ n(\widehat{\mathcal{T}}_{R}),
n(\widehat{\mathcal{T}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{T}}_{R}) \cap
C(\widehat{\mathcal{T}}_{I}^{-1}))=  n(\widehat{\mathcal{T}}_{R})
\geq 2,
\]
and the theorem is proved.
\end{proof}


Finally, we observe that the solutions in Theorem \ref{theorem3.2}
also make sense when $-1<\sigma<0$. In this case, Theorem
\ref{theorem5.1}, \eqref{5.7}, and  the Comparison Theorem imply
that the operator $\mathcal{T}_{3}$, defined in $L^2_{\rm per}([0,L])$
with domain $H^2_{\rm per}([0,L])$, has at least two negative
eigenvalues and so $\mathcal{T}_{R}$ has at least three negative
eigenvalues. Thus, we can establish the following.

\begin{theorem}  \label{theorem5.5}
Let  $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
$\ref{theorem3.2}$ and $-1<\sigma<0$. Then the orbit
$$
\widetilde{\mathcal{O}}=\{T_p(\gamma
s)(\phi_{\gamma},\phi_\gamma,0,0); s \in {\mathbb{R}} \}
$$
is $X_1$-unstable, in the sense of Definition \ref{definition6.1},
by the periodic flow of system $(\ref{1.4})$.
\end{theorem}

\begin{proof}
As we have already pointed out, the advantage in using the Grillakis
approach is that we do not need to know the exact number of negative
eigenvalues of the operator $\mathcal{T}_{\gamma}$, but only to have
an estimate of the number in \eqref{5.10}. In the present case,
since
\[
\max\{ n(\widehat{\mathcal{T}}_{R}),
n(\widehat{\mathcal{T}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{T}}_{R}) \cap
C(\widehat{\mathcal{T}}_{I}^{-1}))=  n(\widehat{\mathcal{T}}_{R})
\]
we can use the estimate (see e.g. \cite{Angulo5} or
\cite{Pelinovsky1})
\begin{equation}   \label{5.11}
 n(\widehat{\mathcal{T}}_{R}) \geq  n(\mathcal{T}_{R})- \dim
 (\ker(\mathcal{T}_{I}))
\end{equation}
 to conclude that
 \[
\max\{ n(\widehat{\mathcal{T}}_{R}),
n(\widehat{\mathcal{T}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{T}}_{R}) \cap
C(\widehat{\mathcal{T}}_{I}^{-1})) \geq 3-2=1.
\]
This proves the theorem.
\end{proof}


\begin{remark} \label{rmk} \rm
The estimate \eqref{5.11} may be proved following similar
arguments as in the proof of \cite[Theorem 3.2]{Grillakis1} (see
also \cite{Yew}).
\end{remark}


\subsection*{Acknowledgments}
The author was supported by grant no. 152234/2007-1 from CNPq-Brazil.
The author thanks Professor Felipe Linares for reading the first
draft of the paper. The author also thanks the referee for the
suggestions, which improved the presentation of the paper.

\begin{thebibliography}{99}

\bibitem{Agrawal} G. Agrawal;
\emph{Nonlinear Fiber Optics}, Academic Press, Boston, (1989).

\bibitem{Ambrosetti} A. Ambrosetti and  E. Colorado;
Bound and ground states of coupled nonlinear Schr\"odinger
equations, \emph{C. R.
Acad. Sci. Paris, Ser. I} \textbf{342} (2006), 453--458.

\bibitem{Angulo1} J. Angulo;
  Stability of periodic travelling waves solutions
to the Schr\"{o}dinger and modi\-fied  Korteweg-de Vries equations,
\emph{J. Differential Equations} \textbf{235} (2007), 1-30.

\bibitem{Angulo2} J. Angulo and F. Linares;
  Periodic pulses of
coupled nonlinear Schr\"{o}dinger equations in optics, \emph{Indiana
Univ. Math. J.} \textbf{56} (2007), 847--878.

\bibitem{Angulo5} J. Angulo and A. Pastor;
  Stability of periodic optical
solitons for a nonlinear Schr\"odinger system,  \emph{Proc. Roy. Soc.
Edinburgh Sect. A} \textbf{139} (2009), 927--959.


\bibitem{Bourgain} J. Bourgain;
 Fourier transform restriction
phenomena for certain lattice subsets and applications to nonlinear
evolution equations I, II,  \emph{Geom.  Funct. Anal.} \textbf{3}
(1993), 107--156, 209--262.

\bibitem{Bourgain1} J. Bourgain;
  \emph{Global Solutions of Nonlinear
Schr\"{o}dinger Equations}, Amer. Math. Soc. Coll. Publ., \textbf{46},
American Mathematical Society, Providence (1999).

\bibitem{Byrd} P. F. Byrd and M. D. Friedman;
 \emph{Handbook of
Elliptic Integrals for Engineers and Scientists}, Springer Verlag,
Nova York (1971).

\bibitem{deFigueiredo} D. G. de Figueiredo and O. Lopes;
Solitary waves for some nonlinear Schr\"odinger systems, \emph{Ann.
Inst. H. Poincar\'e Anal. Non Lin\'eaire} \textbf{25} (2008),
149--161.

\bibitem{Eastham} M.S.P. Eastham;
\emph{The Spectral Theory of
Periodic Differential Equtions}, Scottish Academic Press, London
(1973).

\bibitem{Grillakis1} M. Grillakis;
Linearized instability for
nonlinear Schr\"{o}dinger and Klein-Gordon equations, \emph{Comm.
Pure Appl. Math.} \textbf{41} (1988), 747--774.

\bibitem{Grillakis2} M. Grillakis;
Analysis of the linearization around a critical point of an
infinite-dimensional Hamiltonian system,
\emph{Comm. Pure Appl. Math.} \textbf{43} (1990),
299--333.

\bibitem{Grillakis4} M. Grillakis, J. Shatah, and W. Strauss;
Stability theory of solitary waves in the presence of symmetry
II, \emph{J. Funct. Anal.} \textbf{94} (1990), 308--348.

\bibitem{Ince} E. L. Ince;
The periodic Lam\'e functions,
\emph{Proc. Roy. Soc. Edinburgh} \textbf{60} (1940), 47--63.

\bibitem{Kenig2} C. E. Kenig, G. Ponce, and L. Vega;
 Quadratic forms for the 1-D semilinear Schr\"{o}dinger equation,
\emph{Trans. Amer. Math. Soc.} \textbf{348} (1996), 3323--3353.

\bibitem{Kenig3} C. E. Kenig, G. Ponce, and L. Vega;
The Cauchy problem for the Korteweg-de Vries equation in Sobolev spaces of
negative indices, \emph{Duke Math. J.}  \textbf{71} (1993), 1--21.

\bibitem{Kivshar} Y. S. Kivshar;
 Bright and dark spatial solitons in non-Kerr media,
\emph{J. Opt. Quantum Electron.} \textbf{30} (1998), 571--614.

\bibitem{Kivshar2} Y. S. Kivshar and B. Luther-Davies;
 Dark optical solitons: physics and applications,
 \emph{Physics Reports} \textbf{298} (1998), 81--197.

\bibitem{Lin} T. C. Lin and J. Wei;
 Ground states of $N$ coupled nonlinear Schr\"odinger equations
in $\mathbb{R}^n, n\leq 3$, \emph{Commun. Math. Phys.} \textbf{255} (2005), 629--653.

\bibitem{Lin1} T. C. Lin and J. Wei, Erratum;
Ground states of $N$ coupled nonlinear Schr\"odinger equations
in $\mathbb{R}^n, n\leq 3$,
\emph{Commun. Math. Phys.} \textbf{277} (2008), 573--576.

\bibitem{Magnus} W. Magnus and Winkler;
\emph{Hill's Equation}, vol \textbf{20}, Interscience,
Tracts in Pure and Appl. Math., 1976.

\bibitem{Menyuk} C. G. Menyuk;
Pulse propagation in an elliptically birefringent Kerr medium,
\emph{IEEE. J. Quantum Electron}  \textbf{25} (1989) 2674--2682.


\bibitem{Ohta} M. Ohta;
Stability of solitary waves for coupled nonlinear
Schr\"odinger equations, \emph{Nonlinear Anal.} \textbf{26} (1996),
933--939.

\bibitem{Pastor} A. Pastor;
Nonlinear and spectral stability of periodic travelling wave
solutions for a nonlinear Schr\"{o}dinger system, \emph{Differential
Integral Equations} \textbf{23} (2010), 125--154.

\bibitem{Pelinovsky}  D. E. Pelinovsky and Y. S. Kivshar;
Stability criterion for multicomponent solitary waves,
\emph{Phys. Rev. E} \textbf{62} (2000), 8668--8676.

\bibitem{Pelinovsky1}  D. E. Pelinovsky;
Inertia law for spectral stability of solitary waves in
coupled nonlinear Schr\"odinger equations,
\emph{Proc. Roy. Soc. Edinburgh Sect. A} \textbf{461} (2005),
783--812.

\bibitem{Sammut1} A. R. Sammut, A. V. Buryak, and Y. S. Kivshar;
 Modification of solitary waves by third-harmonic generation,
 \emph{Opt. Lett.} \textbf{22} (1997), 1385--1387.

\bibitem{Sammut} A. R. Sammut, A. V. Buryak, and Y. S. Kivshar;
Bright and dark solitary waves in the presence of the third-harmonic
generation, \emph{J. Opt. Soc. Am. B} \textbf{15} (1998), 1488--1496.

\bibitem{Shatah} J. Shatah and W. Strauss;
 Spectral condition for instability,
\emph{Contemp. Math.} \textbf{255} (2000), 189--198.

\bibitem{Sirakov} B. Sirakov;
 Least energy solitary waves for a system of nonlinear
schr\"odinger equations in $\mathbb{R}^n$, \emph{Commun. Math. Phys.}
\textbf{271} (2007), 199--221.

\bibitem{Yew} A. C. Yew;
 Stability analysis of multipulses
in nonlinearly-coupled Schr\"{o}dinger equations, \emph{Indiana Univ.
Math. J.} \textbf{49} (2000), 1079--1124.

\end{thebibliography}

\end{document}