\documentclass[reqno]{amsart} \usepackage{hyperref} \usepackage[compress]{cite} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2015 (2015), No. 127, pp. 1--11.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2015 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2015/127\hfil Super-quadratic conditions] {Super-quadratic conditions for periodic elliptic system on $\mathbb{R}^N$} \author[F. Liao, X. Tang, J. Zhang, D. Qin \hfil EJDE-2015/127\hfilneg] {Fangfang Liao, Xianhua Tang, Jian Zhang, Dongdong Qin} \address{Fangfang Liao \newline School of Mathematics and Statistics, Central South University, Changsha, 41008, 3 Hunan, China. \newline School of Mathematics and Finance, Xiangnan University, Chenzhou, 423000, Hunan, China} \email{liaofangfang1981@126.com} \address{Xianhua Tang (corresponding author) \newline School of Mathematics and Statistics, Central South University, Changsha, 41008, 3 Hunan, China} \email{tangxh@mail.csu.edu.cn} \address{Jian Zhang \newline School of Mathematics and Statistics, Central South University, Changsha, 41008, 3 Hunan, China} \email{zhangjian433130@163.com} \address{Dongdong Qin \newline School of Mathematics and Statistics, Central South University, Changsha, 41008, 3 Hunan, China} \email{qindd132@163.com} \thanks{Submitted January 31, 2015. Published May 6, 2015.} \subjclass[2010]{35J10, 35J20} \keywords{Elliptic system; super-quadratic; nontrivial solution; \hfill\break\indent strongly indefinite functionals} \begin{abstract} This article concerns the elliptic system \begin{gather*} -\Delta u+V(x)u=W_{v}(x, u, v), \quad x\in \mathbb{R}^{N},\\ -\Delta v+V(x)v=W_{u}(x, u, v), \quad x\in \mathbb{R}^{N},\\ u, v\in H^{1}(\mathbb{R}^{N}), \end{gather*} where $V $ and $W$ are periodic in $x$, and $W(x, z)$ is super-linear in $z=(u, v)$. We use a new technique to show that the above system has a nontrivial solution under concise super-quadratic conditions. These conditions show that the existence of a nontrivial solution depends mainly on the behavior of $W(x, u, v)$ as $|u+v| \to 0$ and $|au+bv| \to \infty$ for some positive constants $a, b$. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{remark}[theorem]{Remark} \newtheorem{example}[theorem]{Example} \allowdisplaybreaks \section{Introduction} In this article, we study the elliptic system \begin{equation} \label{hs} \begin{gathered} -\Delta u+V(x)u=W_{v}(x, u, v), \quad x\in \mathbb{R}^{N},\\ -\Delta v+V(x)v=W_{u}(x, u, v), \quad x\in \mathbb{R}^{N},\\ u, v\in H^{1}(\mathbb{R}^{N}), \end{gathered} \end{equation} where $z:=(u, v) \in \mathbb{R}^2$, $V : \mathbb{R}^{N} \to {\mathbb{R}}$ and $W: \mathbb{R}^N\times \mathbb{R}^2 \to {\mathbb{R}}$. Systems similar to \eqref{hs} have been considered recently; see for instance \cite{ACM,AY,AY2,BF,DL,FYD,LTZ,LY,PR,S,SS,WXZ1,WXZ2,YCD,ZCZ,ZD,ZTZ,ZTZ1, ZQZ,ZZ,ZZD1,ZZD2,ZZD3,ZZD4} and references therein. For the superquadratic case, it always assumed that $W$ satisfies the Ambrosetti-Rabinowitz condition \begin{itemize} \item[(AR)] there is a $\mu>2$ such that \begin{equation}\label{AR1} 0< \mu W(x, z)\le W_z(x, z)\cdot z, \quad \forall (x, z)\in \mathbb{R}^{N}\times \mathbb{R}^2, \; z\ne 0\,. \end{equation} \end{itemize} We use the assumption that there exist $c>0$ and $\nu\in (2N/(N+2), 2)$ such that \begin{equation}\label{AR2} |W_z(x, z)|^{\nu} \le c[1+W_z(x, z)\cdot z], \quad \forall (x, z)\in \mathbb{R}^{N}\times \mathbb{R}^2\,, \end{equation} or the super-quadratic condition \begin{equation}\label{AR3} \lim_{|z|\to \infty}\frac{|W(x, z)|}{|z|^2}=\infty, \quad \text{uniformly in }x\in \mathbb{R}^N, \end{equation} and a condition of the Ding-Lee type, \begin{itemize} \item[(DL)] $\widetilde{W}(x, z):=\frac{1}{2}W_z(x, z)\cdot z-W(x, z)> 0$ for $z\ne 0$ and there exist $\hat{c}>0$ and $\kappa>\max\{1, N/2\}$ such that \begin{equation}\label{AR4} |W_z(x, z)|^{\kappa}\le \hat{c}|z|^{\kappa}\widetilde{W}(x, z), \quad \text{for large } |z|. \end{equation} \end{itemize} Observe that conditions \eqref{AR3} and $W(x, z)>0, \forall z\ne 0$ in (AR) or $\widetilde{W}(x, z)>0, \forall\ z\ne 0$ in (DL) play an important role for showing that any Palais-Smale sequence or Cerami sequence is bounded in the aforementioned works. However, there are many functions that do not satisfy these conditions, for example, $$ W(x, u, v)=(u^2+uv+v^2)\ln(1+u^2), $$ or $$ W(x, u, v)=(u+2v)^2\sqrt{u^2+v^2}. $$ In a recent paper Liao, Tang and Zhang \cite{LTZ} studied the existence of solutions for system \eqref{hs} under the following assumptions on $V$ and $W$: \begin{itemize} \item[(V1)] $V\in C(\mathbb{R}^N, \mathbb{R})$, $V(x)$ is 1-periodic in each of $x_1, x_2, \dots, x_N$, and $\min_{\mathbb{R}^N}V \geq \beta_0>0$; \item[(W1)] $W\in C(\mathbb{R}^N\times \mathbb{R}^2, \mathbb{R}^+)$, $W(x, z)$ is 1-periodic in each of $x_1, x_2, \dots, x_N$, continuously differentiable on $z \in \mathbb{R}^2$ for every $x \in \mathbb{R}^N$, and there exist constants $p\in (2, 2^*)$ and $C_0>0$ such that $$ |W_z(x, z)|\le C_0\left(1+|z|^{p-1}\right), \quad \forall (x, z)\in \mathbb{R}^N\times \mathbb{R}^2; $$ \item[(W2)] $|W_z(x,z)|=o(|z|)$, as $|z|\to 0$, uniformly in $x \in \mathbb{R}^N$; \item[(W3)] $\lim_{|u+v|\to \infty}\frac{|W(x, u, v)|}{|u+v|^2}=\infty$, a.e. $x\in \mathbb{R}^N$; \item[(W4)] $\widetilde{W}(x, z) \ge 0$ for all $(x, z) \in \mathbb{R}^{N}\times \mathbb{R}^2$, and there exist $c_0, R_0>0$ and $\kappa>\max\{1, N/2\}$ such that $$ |W_{u}(x, u, v)+W_{v}(x, u, v)|\le \frac{2\beta_0}{3}\sqrt{u^2+v^2}, \quad u^2+v^2\le R_0^2 $$ and $$ |W_{u}(x, u, v)+W_{v}(x, u, v)|^{\kappa}\le c_0\left(u^2+v^2\right)^{\kappa/2}\widetilde{W}(x, u, v), \quad \ u^2+v^2\ge R_0^2. $$ \end{itemize} Specifically, Liao, Tang and Zhang \cite{LTZ} established the following theorem. \begin{theorem}[{\cite[Theorem1.2]{LTZ}}] \label{thm1.1} Assume that {\rm (V1), (W1)--(W4)} are satisfied. Then \eqref{hs} has a nontrivial solution. \end{theorem} As shown in \cite{LTZ}, (W3) is different from usual superquadratic conditions (AR) and \eqref{AR3}, and is weaker than \eqref{AR3}. Clearly, (W4) is significantly weaker than (DL). By a variable substitution, instead of (W3) and (W4), the following more general conditions were used: \begin{itemize} \item[(W3')] there exist $a, b>0$ such that \[ \lim_{|au+bv|\to \infty}\frac{|W(x, u, v)|}{|au+bv|^2}=\infty, \quad\text{a.e. }x\in \mathbb{R}^N; \] \item[(W4')] $\widetilde{W}(x, z)\ge 0$ for all $(x, z)\in \mathbb{R}^{N}\times \mathbb{R}^2$, and there exist $c_1, R_1>0$ and $\kappa>\max\{1, N/2\}$ such that \begin{gather*} |bW_{u}(x, u, v)+aW_{v}(x, u, v)|\le \frac{2\beta_0}{3}\sqrt{u^2+v^2}, \quad \ a^2u^2+b^2v^2\le R_1^2, \\ |bW_{u}(x, u, v)+aW_{v}(x, u, v)|^{\kappa} \le c_1\left(a^2u^2+b^2v^2\right)^{\kappa/2}\widetilde{W}(x, u, v),\\ a^2u^2+b^2v^2\ge R_1^2. \end{gather*} \end{itemize} Motivated by \cite{LTZ}, we obtain a super-quadratic condition more concise than (W4'): \begin{itemize} \item[(W5)] $\widetilde{W}(x, z)\ge 0$ for all $(x, z)\in \mathbb{R}^{N}\times \mathbb{R}^2 $, and there exist $\theta \in (0, 1)$, $\alpha_0 > 0$, and $\kappa>\max\{1, N/2\}$ such that \begin{align*} &\frac{|bW_u(x, z)+aW_v(x, z)|}{|z|} \geq \theta \beta_0 \min\{a, b\}\\ & \Rightarrow \Big(\frac{|bW_u(x, z)+aW_v(x, z)|}{|z|}\Big)^\kappa \le \alpha_0 \widetilde{W}(x, z). \end{align*} \end{itemize} By introducing new techniques, under (W3') and (W5), we obtain the linking structure and the boundedness of a Cerami sequence of the energy functional associated with \eqref{hs}. Specifically, we obtain the following theorem. \begin{theorem} \label{thm1.2} Assume that{\rm (V1), (W1), (W2), (W3'), (W5)} hold. Then \eqref{hs} has a nontrivial solution. \end{theorem} \begin{remark} \label{rmk1.3}\rm Note that (W5) is weaker than (DL) and than (AR). Since \[ |bW_{u}+aW_{v}|\le \sqrt{a^2+b^2}|W_{z}(x, z)|, \] in view of (W2), it is clear that (DL) implies (W5). If $W(x, z)$ satisfies \eqref{AR1}, then there exist $c_1, R_1>0$ such that \begin{gather}\label{Re1} W_z(x, z)\cdot z\ge \mu W(x, z)\ge c_1|z|^{\mu}, \quad |z|\ge R_1,\\ \label{Re2} \widetilde{W}(x, z)\ge \frac{\mu-2}{2}W_z(x, z)\cdot z>0, \quad \forall z\in \mathbb{R}^2\setminus \{0\}. \end{gather} Let $\kappa=\nu/(2-\nu)$. Then $\kappa>\max\{1, N/2\}$. Hence, it follows from \eqref{AR2}, \eqref{Re1} and \eqref{Re2} that \begin{equation} \label{Re3} \begin{aligned} |W_z(x, z)|^{\kappa} &\leq c_2|W_z(x, z)|^{\kappa-\nu}W_z(x, z)\cdot z \\ &\leq c_3|z|^{(\kappa-\nu)/(\nu-1)}\widetilde{W}(x, z) \\ &= c_3|z|^{\kappa}\widetilde{W}(x, z), \quad |z|\ge R_1. \end{aligned} \end{equation} This shows that (DL) holds, and so (W5) holds. \end{remark} Before proceeding with the proof of Theorem \ref{thm1.2}, we give a nonlinear example to illustrate the assumptions. \begin{example} \label{examp1.4}\rm $W(x, u, v)=h(x)(u+2v)^2\sqrt{u^2+v^2}$, where $h\in C(\mathbb{R}^N, (0, \infty))$ is 1-periodic in each of the variables $x_1, x_2, \ldots, x_N$. Then \[ \widetilde{W}(x, u, v)=\frac{1}{2}h(x)(u+2v)^2\sqrt{u^2+v^2}, \quad u, \, v\in \mathbb{R}. \] Therefore all conditions (W1), (W2), (W3'), (W5) are satisfied with $a=1, b=2$ and $\kappa\le 3$. Note that $W(x, u, v)=\widetilde{W}(x, u, v)=0$ for $u=-2v$, $v\in \mathbb{R}$, thus $W$ does not satisfy (AR) and (DL). \end{example} The rest of this article is organized as below. In Section 2, we provide a variational setting. The proofs of our main results are given in the last section. \section{Variational setting} Under assumption (V1), we can define the Hilbert space $$ E_V=\big\{u \in H^1(\mathbb{R}^N): \int_{\mathbb{R}^N} (|\nabla u|^2+V(x)u^2)\,\mathrm{d}x < +\infty \big\} $$ equipped with the inner product $$ (u, v)_{E_V}=\int_{\mathbb{R}^N} [\nabla{u}\cdot \nabla{v} +V(x)uv]\,\mathrm{d}x, \quad \forall u, v\in E_V, $$ and the corresponding norm \begin{equation}\label{2.1-} \|u\|_{E_V}=\Big(\int_{\mathbb{R}^N} [|\nabla{u}|^2+V(x)u^2]\,\mathrm{d}x\Big) ^{1/2}, \quad \forall u\in E_V. \end{equation} By the Sobolev embedding theorem, there exists constant $\gamma_s>0$ such that \begin{equation}\label{2.1} \|u\|_{s}\le \gamma_s\|u\|_{E_V}, \quad \forall u\in H^1(\mathbb{R}^N), \; 2\le s \le 2^*, \end{equation} here and in the sequel, by $\|\cdot\|_s$ we denote the usual norm in space $L^s(\mathbb{R}^N)$. Let $E =E_V\times E_V $ with the inner product \[ (z_1,z_2)=((u_1, v_1), (u_2, v_2)) = (u_1, u_2)_{E_V} + (v_1, v_2)_{E_V}, \] for $z_i=(u_i, v_i)\in E$, $i=1, 2$, and the corresponding norm $\|\cdot\|$. Then there hold \begin{equation}\label{2.2} \|z\|^2=\|u\|_{{E_V}}^2+\|v\|_{{E_V}}^2, \quad \forall z=(u, v)\in E \end{equation} and \begin{equation} \label{2.3} \begin{aligned} \|z\|_s^s &= \int_{\mathbb{R}^N}(u^2+v^2)^{s/2}\,\mathrm{d}x \le 2^{(s-2)/2}(\|u\|_s^s+\|v\|_s^s) \\ &\leq 2^{(s-2)/2}\gamma_s^{s}\left(\|u\|_{E_V}^s+\|v\|_{E_V}^s\right)\\ &\le 2^{(s-2)/2}\gamma_s^{s}\left(\|u\|_{E_V}^2+\|v\|_{E_V}^2\right)^{s/2} \\ &= 2^{(s-2)/2}\gamma_s^{s}\|z\|^s, \quad \forall s\in [2, 2^*], \; z=(u, v)\in E. \end{aligned} \end{equation} Now we define a functional $\Phi$ on $E$ by \begin{equation}\label{2.4} \Phi(z)=\int_{\mathbb{R}^N}\left(\nabla u\cdot\nabla v+V(x)uv\right) \,\mathrm{d}x -\int_{\mathbb{R}^N}W(x, u, v)\,\mathrm{d}x, \quad \forall z=(u, v)\in E. \end{equation} Consequently, under assumptions (V1), (V2), (W1), (W2), (W3'), it is well known that $\Phi$ is $C^1(E, \mathbb{R})$, and \begin{equation}\label{2.5} \begin{aligned} \langle \Phi'(z), \zeta \rangle &= \int_{\mathbb{R}^N}\left[\nabla u\cdot\nabla \psi +\nabla v\cdot\nabla \varphi+V(x)(u\psi+v\varphi)\right]\,\mathrm{d}x \\ & \quad -\int_{\mathbb{R}^N}[W_{u}(x, u, v)\varphi +W_{v}(x, u, v)\psi]\,\mathrm{d}x, \end{aligned} \end{equation} for all $z=(u, v)$, $\zeta=(\varphi, \psi)\in E$. Let $$ E^{-}=\{(u, -u) : u\in H^1(\mathbb{R}^N)\}, \quad E^{+}=\{(u, u) : u\in H^1(\mathbb{R}^N)\}. $$ For any $z=(u, v)\in E$, set \begin{equation}\label{2.6} z^{-}=\Big(\frac{u-v}{2}, \frac{v-u}{2}\Big), \quad z^{+}=\Big(\frac{u+v}{2}, \frac{u+v}{2}\Big). \end{equation} It is obvious that $z=z^{-}+z^{+}$, $z^{-}$ and $z^{+}$ are orthogonal with respect to the inner products $(\cdot, \cdot)_{L^2}$ and $(\cdot, \cdot)$. Thus we have $E=E^{-}\oplus E^{+}$. By a simple calculation, one can get that $$ \frac{1}{2}\left(\|z^{+}\|^2-\|z^{-}\|^2\right) =\int_{\mathbb{R}^N}\left[\nabla u\cdot \nabla v+V(x)uv\right]\,\mathrm{d}x. $$ Therefore, the functional $\Phi$ defined in \eqref{2.4} can be rewritten in a standard way \begin{equation}\label{2.7} \Phi(z)=\frac{1}{2}\left(\|z^{+}\|^2-\|z^{-}\|^2\right) -\int_{\mathbb{R}^N}W(x, z)\,\mathrm{d}x, \quad \forall z=(u, v)\in E. \end{equation} Moreover \begin{equation}\label{2.8} \langle \Phi'(z), z \rangle = \|z^{+}\|^2-\|z^{-}\|^2 -\int_{\mathbb{R}^N}[W_{u}(x, u, v)u +W_{v}(x, u, v)v]\,\mathrm{d}x, \end{equation} for all $z=(u, v)\in E$. \section{Proofs of main resutls} To give the proofs of our results, we set \begin{equation}\label{3.1} \Psi(z) = \int_{\mathbb{R}^N} W(x, z) \,\mathrm{d}x, \quad \forall z \in E. \end{equation} \begin{lemma} \label{lem3.1} Suppose that {\rm (W1), (W2)} are satisfied. Then $\Psi$ is nonnegative, weakly sequentially lower semi-continuous, and $\Psi'$ is weakly sequentially continuous. \end{lemma} Using the Sobolev's imbedding theorem, one can easily check the above lemma, so we omit the proof. \begin{lemma} \label{lem3.2} Suppose that {\rm (V1), (W1), (W2), (W3')} are satisfied. Then there is a $\rho>0$ such that $\kappa_1:=\inf\Phi(S_{\rho}^{+})>0$, where $S_{\rho}^{+}=\partial B_{\rho}\cap E^{+}$. \end{lemma} The above lemma can be proved in standard way; we omit its proof. \begin{lemma} \label{lem3.3} Suppose that {\rm (V1), (W1), (W2), (W3')} are satisfied. Let $e=(e_0, e_0)$ belong to $E^{+}$ with $\|e\|=1$. Then there is a constant $r>0$ such that $\sup \Phi(\partial Q)\le 0$, where \begin{equation}\label{3.2} Q=\big\{\zeta+se : \zeta=(w, -w)\in E^{-},\, s\ge 0,\, \|\zeta+se\|\le r\big\}. \end{equation} \end{lemma} \begin{proof} By (W1) and \eqref{2.7}, $\Phi(z)\le 0$ for $z\in E^{-}$. Next, it is sufficient to show that $\Phi(z)\to -\infty$ as $z\in E^{-}\oplus \mathbb{R} e$ for $\|z\|\to \infty$. Arguing indirectly, assume that for some sequence $\{\zeta_n+s_ne\}\subset E^{-}\oplus \mathbb{R} e$ with $\|\zeta_n+s_ne\|\to \infty$, there is $M>0$ such that $\Phi(\zeta_n+s_ne)\ge -M$ for all $n\in \mathbb{N}$. Set $\zeta_n=(w_n, -w_n)$, $\xi_n=(\zeta_n+s_ne)/\|\zeta_n+s_ne\| =\xi_n^{-}+t_ne$, then $\|\xi_n^{-}+t_ne\|=1$. Passing to a subsequence, we may assume that $t_n \to \bar{t}$ and $\xi_n\rightharpoonup \xi$ in $E$, then $\xi_n\to \xi$ a.e. on $\mathbb{R}^N$, $\xi_n^{-}\rightharpoonup \xi^{-}$ in $E$, $\xi_n^-:=(\tilde{w_n}, -\tilde{w_n})\rightharpoonup \xi^-:=(\tilde{w}, -\tilde{w})$, and \begin{equation}\label{3.3} \begin{aligned} -\frac{M}{\|\zeta_n+s_ne\|^2} &\le \frac{\Phi(\zeta_n+s_ne)}{\|\zeta_n+s_ne\|^2}\\ &=\frac{t_n^2}{2}-\frac{1}{2}\|\xi_n^{-}\|^2 -\int_{\mathbb{R}^N} \frac{W(x, w_n+s_ne_0, -w_n+s_ne_0)}{\|\zeta_n+s_ne\|^2}\,\mathrm{d}x. \end{aligned} \end{equation} If $\bar{t}=0$, then it follows from \eqref{3.3} that $$ 0\le \frac{1}{2}\|\xi_n^{-}\|^2+\int_{\mathbb{R}^N} \frac{W(x, w_n+s_ne_0, -w_n+s_ne_0)}{\|\zeta_n+s_ne\|^2}\,\mathrm{d}x \le \frac{t_n^2}{2}+\frac{M}{\|\zeta_n+s_ne\|^2}\to 0, $$ which yields $\|\xi_n^{-}\|\to 0$, and so $1=\|\xi_n\|\to 0$, a contradiction. If $\bar{t}\ne 0$, then \begin{equation}\label{3.4} (a-b)\tilde{w}+(a+b)\bar{t} e_0 \neq 0. \end{equation} Arguing indirectly, assume that $(a-b)\tilde{w}+(a+b)\bar{t} e_0 =0$, then $a \neq b$ and \begin{align*} \bar{t}^2 &= \lim_{n \to \infty} t_n^2 \\ &\geq \lim_{n \to \infty}\inf \Big( -\frac{2M}{\|\zeta_n+s_ne\|^2}+\|\xi_n^-\|^2 \Big)\\ & \geq \|\xi^-\|^2\\ &= \int_{\mathbb{R}^N} [|\nabla \xi^-|^2+V(x)|\xi^-|^2 ]\,\mathrm{d}x\\ &= \frac{(a+b)^2}{(b-a)^2}\bar{t}^2 \int_{\mathbb{R}^N}\left[|\nabla e|^2+V(x)|e|^2 \right]\,\mathrm{d}x\\ & > \bar{t}^2 \int_{\mathbb{R}^N} [|\nabla e|^2+V(x)|e|^2 ]\,\mathrm{d}x = \bar{t}^2, \end{align*} which is a contradiction. Let $\Omega := \{x \in \mathbb{R}^N: (a-b)\tilde{w}(x)+(a+b)\bar{t}e_0(x)\neq 0\}$. Then \eqref{3.4} shows that $|\Omega|>0$. Since $\|\zeta_n+s_ne\| \to \infty$, for any $x \in \Omega$, one has \begin{align*} & |a(w_n(x)+s_ne_0(x))+b(-w_n(x)+s_ne_0(x))|\\ &= \|\zeta_n+s_ne\||(a-b)\tilde{w}_n(x)+(a+b)t_ne_0(x)|\to \infty. \end{align*} Let $\eta_n := a(\tilde{w}_n+t_ne_0)+b(-\tilde{w}_n+t_ne_0)$. It follows from \eqref{3.3}, \eqref{3.4}, (W3') and Fatou's lemma that \begin{align*} 0 &\leq \limsup_{n\to\infty} \Big[\frac{t_n^2}{2}-\frac{1}{2}\|\xi_n^{-}\|^2 -\int_{\mathbb{R}^N}\frac{W(x, w_n+s_ne_0, -w_n+s_ne_0)} {\|\zeta_n+s_ne\|^2}\,\mathrm{d}x\Big]\\ &= \limsup_{n\to\infty}\Big[\frac{t_n^2}{2}-\frac{1}{2}\|\xi_n^{-}\|^2 -\int_{\mathbb{R}^N}\frac{W(x, w_n+s_ne_0, -w_n+s_ne_0)}{|a(w_n+s_ne_0)+b(-w_n+s_ne_0)|^2}|\eta_n|^2\,\mathrm{d}x\Big]\\ &\leq \frac{1}{2}\lim_{n\to\infty}t_n^2-\liminf_{n\to\infty} \int_{\mathbb{R}^N}\frac{W(x, w_n+s_ne_0, -w_n+s_ne_0)} {|a(w_n+s_ne_0)+b(-w_n+s_ne_0)|^2}|\eta_n|^2\,\mathrm{d}x\\ &\leq \frac{\bar{t}^2}{2}-\int_{\mathbb{R}^N} \liminf_{n\to\infty} \frac{W(x, w_n+s_ne_0, -w_n+s_ne_0)} {|a(w_n+s_ne_0)+b(-w_n+s_ne_0)|^2}|\eta_n|^2\,\mathrm{d}x\\ &= -\infty, \end{align*} a contradiction. \end{proof} Applying the generalized linking theorem \cite{KS, LS} and standard arguments, we can prove the following lemma. \begin{lemma} \label{lem3.4} Suppose that {\rm (V1), (W1), (W2), (W3')} are satisfied. Then there exist a constant $c_*\in [\kappa_0, \sup \Phi(Q)]$ and a sequence $\{z_n\}=\{(u_n, v_n)\} \subset E$ satisfying \begin{equation}\label{3.5} \Phi(z_n)\to c_*, \quad \|\Phi'(z_n)\|(1+\|z_n\|)\to 0. \end{equation} where $Q$ is defined by \eqref{3.2}. \end{lemma} \begin{lemma} \label{lem3.5} Suppose that {\rm (V1), (W1), (W2), (W3'), (W5)} are satisfied. Then any sequence $\{z_n\}=\{(u_n, v_n)\} \subset E$ satisfying \eqref{3.5} is bounded in $E$. \end{lemma} \begin{proof} To prove the boundedness of $\{z_n\}$, arguing by contradiction, suppose that $\|z_n\| \to \infty$. Let \begin{gather*} \xi_n=\frac{z_n}{\|z_n\|}=(\varphi_n, \psi_n), \quad \hat{z}_n=(\hat{u}_n, \hat{v}_n) := \Big(\frac{au_n+bv_n}{2a}, \frac{au_n+bv_n}{2b} \Big), \\ \hat{\xi}_n=(\hat{\varphi}_n, \hat{\psi}_n) :=\frac{\hat{z}_n}{\|z_n\|} =\Big(\frac{a\varphi_n+b\psi_n}{2a}, \frac{a\varphi_n+b\psi_n}{2b}\Big). \end{gather*} By (W1), \eqref{2.2}, \eqref{2.4}, \eqref{2.7}, \eqref{2.8} and \eqref{3.5}, one obtains \begin{gather}\label{3.6} 2c_*+o(1) = \|z_n^{+}\|^2-\|z_n^{-}\|^2-2\int_{\mathbb{R}^N}W(x, z_n) \,\mathrm{d}x\le\|z_n^{+}\|^2-\|z_n^{-}\|^2, \\ \label{3.7} c_*+o(1) = \int_{\mathbb{R}^N}\widetilde{W}(x, z_n)\,\mathrm{d}x, \end{gather} and \begin{align*} \|\hat{z}_n\|^2 &= \frac{a^2+b^2}{4a^2b^2}\|au_n+bv_n\|_{E_V}^2 \\ &= \frac{a^2+b^2}{4a^2b^2}\Big[a^2\|u_n\|_{E_V}^2+b^2\|v_n\|_{E_V}^2 +2ab \int_{\mathbb{R}^N}(\nabla u_n\nabla v_n+V(x)u_nv_n)\Big] \\ &= \frac{a^2+b^2}{4a^2b^2}\Big[a^2\|u_n\|_{E_V}^2+b^2\|v_n\|_{E_V}^2 +2ab\Big(\Phi(z_n)+\int_{\mathbb{R}^N}W(x, u_n, v_n)\,\mathrm{d}x\Big)\Big] \\ & \geq \frac{a^2+b^2}{4a^2b^2} [\min\{a^2, b^2\}\|z_n\|^2+2ab(c_*+o(1))], \end{align*} which implies \begin{equation}\label{3.8} \|z_n\| \leq \frac{2ab}{\sqrt{a^2+b^2}\min\{a,b\}}\|\hat{z}_n\|, \quad a, b>0. \end{equation} Note that \begin{equation}\label{b4} \begin{aligned} \|\hat{\xi}_n\|^2 &= \frac{a^2+b^2}{4a^2b^2}\|a\varphi_n+b\psi_n\|_{E_V}^2 \\ & \leq \frac{a^2+b^2}{4a^2b^2} \left(a\|\varphi_n\|_{E_V}+b\|\psi_n\|_{E_V}\right)^2 \\ & \leq \frac{a^2+b^2}{2a^2b^2} \left(a^2\|\varphi_n\|_{E_V}^2+b^2\|\psi_n\|_{E_V}^2\right) \\ & \leq \frac{(a^2+b^2)^2}{2a^2b^2} \left(\|\varphi_n\|_{E_V}^2+\|\psi_n\|_{E_V}^2\right) \\ &= \frac{(a^2+b^2)^2}{2a^2b^2}\|\xi_n\|^2=\frac{(a^2+b^2)^2}{2a^2b^2}, \end{aligned} \end{equation} which implies that $\{\hat{\xi}_n\}$ is bounded. If $\delta:= \lim\sup_{n \to \infty} \sup_{y \in \mathbb{R}^N} \int_{B(y,1)}|\hat{\xi_n}|^2\,\mathrm{d}x=0$, then by Lions's concentration compactness principle \cite[Lemma 1.21]{Wi}, $a\varphi_n + b\psi_n \to 0$ in $L^s(\mathbb{R}^N)$ for $2 \frac{\delta}{2}$. Since $|\hat{\xi}_n|^2=\frac{a^2+b^2}{4a^2b^2}|a\varphi_n+b\psi_n|^2$, one can get that $$ \int_{B_{1+\sqrt{N}}(k_n)}|a\varphi_n+b\psi_n|^2dx > \frac{2a^2b^2}{a^2+b^2}\delta. $$ Let us define $\tilde{\varphi}_n(x)=\varphi_n(x+k_n)$, $\tilde{\psi}_n(x)=\psi_n(x+k_n)$ so that \begin{equation}\label{3.12} \int_{B_{1+\sqrt{N}}(0)}|a\tilde{\varphi}_n+b\tilde{\psi}_n|^2dx > \frac{2a^2b^2}{a^2+b^2}\delta. \end{equation} Now we define $\tilde{u}_n(x)=u_n(x+k_n)$, $\tilde{v}_n(x)=v_n(x+k_n)$, then $\tilde{\varphi}_n=\tilde{u}_n/\|z_n\|$, $\tilde{\psi}_n=\tilde{v}_n/\|z_n\|$. Passing to a subsequence, we have $a\tilde{\varphi}_n(x)+b\tilde{\psi}_n(x) \rightharpoonup a\tilde{\varphi}(x)+b\tilde{\psi}(x)$ in $E_{V}$, $a\tilde{\varphi}_n(x)+b\tilde{\psi}_n(x) \to a\tilde{\varphi}(x)+b\tilde{\psi}(x)$ in $L^{s}_{\rm loc}(\mathbb{R}^{N})$, $2\le s<2^*$ and $a\tilde{\varphi}_n(x)+b\tilde{\psi}_n(x) \to a\tilde{\varphi}(x)+b\tilde{\psi}(x)$ a.e. on $\mathbb{R}^{N}$. Obviously, \eqref{3.12} implies that $a\tilde{\varphi}(x)+b\tilde{\psi}(x) \ne 0$. Since $\|z_n\|\to \infty$, for a.e. $x\in \{y\in \mathbb{R}^N : a\tilde{\varphi}(y)+b\tilde{\psi}(y)\ne 0\}:=\Omega$, we have $$ \lim_{n\to\infty}|a\tilde{u}_n(x)+b\tilde{v}_n(x)| =\lim_{n\to\infty}\|z_n\||a\tilde{\varphi}_n(x)+b\tilde{\psi}_n(x)|= +\infty. $$ By (W3'), \eqref{3.6} and Fatou's lemma, we have \begin{align*} 0 &= \lim_{n\to\infty}\frac{c_*+o(1)}{\|z_n\|^2} = \lim_{n\to\infty}\frac{\Phi(z_n)}{\|z_n\|^2} \\ &= \lim_{n\to\infty}\left[\frac{1}{2}\|\xi_n^+\|^2 -\frac{1}{2}\|\xi_n^-\|^2 -\int_{\mathbb{R}^N}\frac{W(x, z_n)}{\|z_n\|^2}\,\mathrm{d}x\right] \\ &\leq \lim_{n\to\infty} \left[\frac{1}{2}\|\xi_n^+\|^2 -\frac{1}{2}\|\xi_n^-\|^2 -\int_{\mathbb{R}^N}\frac{W(x, z_n)}{|au_n+bv_n|^2}|a\varphi_n +b\psi_n|^2\,\mathrm{d}x\right]\\ &= \lim_{n\to\infty}\left[\frac{1}{2}\|\xi_n\|^2 -\int_{\mathbb{R}^N}\frac{W(x+k_n, \tilde{z}_n)}{|a\tilde{u}_n +b\tilde{v}_n|^2}|a\tilde{\varphi}_n+b\tilde{\psi}_n|^2\,\mathrm{d}x\right]\\ &\leq \frac{1}{2}-\int_{\Omega}\liminf_{n\to\infty} \Big[\frac{W(x, \tilde{u}_n, \tilde{v}_n)} {|a\tilde{u}_n+b\tilde{v}_n|^2}|a\tilde{\varphi}_n +b\tilde{\psi}_n|^2\Big]\,\mathrm{d}x =-\infty, \end{align*} which is a contradiction. Thus $\{z_n\}$ is bounded in $E$. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.2}] Applying Lemmas \ref{lem3.4} and \ref{lem3.5}, we deduce that there exists a bounded sequence $\{z_n\}=\{(u_n, v_n)\}\subset E$ satisfying \eqref{3.5}. Thus there exists a constant $C_2>0$ such that $\|z_n\|_2\le C_2$. By the Lion's concentration compactness principle (\cite{PL} or \cite [Lemma 1.21]{Wi}), one can rule out the case of vanishing. So nonvanishing occurs. Using a standard translation argument, we can obtain a nontrivial solution of \eqref{hs}. \end{proof} \subsection*{Acknowledgments} This work is partially supported by the NNSF of China (No. 11171351), by the Construct Program of the Key Discipline in Hunan Province, and by the Hunan Provincial Innovation Foundation for Postgraduates. \begin{thebibliography}{00} \bibitem{ACM} C. O. Alves, P. C. Carri\~{a}o, O. H. Miyagaki; \emph{On the existence of positive solutions of a perturbed Hamiltonian system in $\mathbb{R}^N$}, J. Math. Anal. Appl. 276 (2) (2002), 673-690. \bibitem{AY} A. I. \'{A}vila, J. Yang; \emph{On the existence and shape of least energy solutions for some elliptic systems}, J. Differential Equations 191 (2) (2003), 348-376. \bibitem{AY2} A. I. \'{A}vila, J. Yang; \emph{Multiple solutions of nonlinear elliptic systems}, NoDEA Nonlinear Differential Equations Appl. 12 (4) (2005,) 459-479. \bibitem{BF} T. Bartsch, D. G. de Figueiredo; \emph{Infinitely many solutions of nonlinear elliptic systems}, in: Progress in Nonlinear Differential Equations and Their Applications, vol. 35, Birkh\"{a}user, Basel$/$Switzerland, 1999, pp. 51-67. \bibitem{BWW} T. Bartsch, Z. Q. Wang, M. Willem; \emph{The dirichlet problem for superlinear elliptic equations}, in: M. Chipot, P. Quittner (Eds.), Handbook of Differential Equations-Stationary Partial Differential Equations, vol. 2, Elsevier, 2005, pp. 1-5 (Chapter 1). \bibitem{DL} Y. H. Ding, F. H. Lin; \emph{Semiclassical states of Hamiltonian systems of Schr\"odinger equations with subcritical and critical nonlinearities}, J. Partial Differential Equations 19 (3) (2006), 232-255. \bibitem{FYD} D. G. de Figueiredo, J. F. Yang; \emph{Decay, symmetry and existence of solutions of semilinear elliptic systems}, Nonlinear Anal. 33 (3) (1998), 211-234. \bibitem{KS} W. Kryszewski, A. Szulkin; \emph{An infinite dimensional Morse theory with applications}, Trans. Amer. Math. Soc. 349 (8) (1997), 3181-3234. \bibitem{PL} P. L. Lions; \emph{The concentration-compactness principle in the calculus of variations. The locally compact cases II}. Ann. Inst. H. Poincar\'{e} Anal. Non Lin\'{e}aire 1 (4) (1984), 223-283. \bibitem{LS} G. B. Li, A. Szulkin; \emph{An asymptotically periodic Schr\"odinger equation with indefinite linear part}, Commun. Contemp. Math. 4 (4) (2002), 763-776. \bibitem{LTZ} F. F. Liao, X. H. Tang, J. Zhang; \emph{Existence of solutions for periodic elliptic system with general superlinear nonlinearity}, Z. Angew. Math. Phys. 2014, DOI 10.1007/s00033-014-0425-6. \bibitem{LY} G. B. Li, J. F. Yang; \emph{Asymptotically linear elliptic systems}, Comm. Partial Differential Equations 29 (5-6) (2004), 925-954. \bibitem{PR} A. Pistoia, M. Ramos; \emph{Locating the peaks of the least energy solutions to an elliptic system with Neumann boundary conditions}, J. Differential Equations 201 (1) (2004), 160-176. \bibitem{Q} D. D. Qin, X. H. Tang, J. Zhang; \emph{Multiple solutions for semilinear elliptic equations with sign-changing potential and nonlinearity}, Electron. J. Differential Equations, vol. 2013, no. 207 (2013), 1-9. \bibitem{QT} D. D. Qin, X. H. Tang; \emph{New conditions on solutions for periodic Schr\"odinger equations with spectrum zero}, Taiwanese J. Math. DOI: 10.11650/tjm.18.2014.4227. \bibitem{S} B. Sirakov; \emph{On the existence of solutions of Hamiltonian elliptic systems in $\mathbb{R}^N$}, Adv. Differential Equations 5 (10-12) (2000), 1445-1464. \bibitem{SS} B. Sirakov, S. H. M. Soares; \emph{Soliton solutions to systems of coupled Schr\"odinger equations of Hamiltonian type}, Trans. Amer. Math. Soc. 362 (11) (2010), 5729-5744. \bibitem{Wi} M. Willem, Minimax Theorems, Birkh\"{a}user, Boston, 1996. \bibitem{WXZ1} J. Wang, J. X. Xu, F. B. Zhang; \emph{Existence and multiplicity of solutions for asymptotically Hamiltonian elliptic systems in $\mathbb{R}^N$}, J. Math. Anal. Appl. 367 (1) (2010), 193-203. \bibitem{WXZ2} J. Wang, J. X. Xu, F. B. Zhang; \emph{Existence of solutions for nonperiodic superquadratic Hamiltonian elliptic systems}, Nonlinear Anal. 72 (3-4) (2010), 1949-1960. \bibitem{YCD} M. B. Yang, W. X. Chen, Y. H. Ding; \emph{Solutions of a class of Hamiltonian elliptic systems in $\mathbb{R}^N$}, J. Math. Anal. Appl. 362 (2) (2010), 338-349. \bibitem{ZCZ} R. M. Zhang, J. Chen, F. K. Zhao; \emph{Multiple solutions for superlinear elliptic systems of Hamiltonian type}, Discrete Contin. Dyn. Syst. 30 (4) (2011), 1249-1262. \bibitem{ZD} F. K. Zhao, Y. H. Ding; \emph{On Hamiltonian elliptic systems with periodic or non-periodic potentials}, J. Differential Equations 249 (12) (2010), 2964-2985. \bibitem{ZQZ} J. Zhang, W. P. Qin, F. K. Zhao; \emph{Existence and multiplicity of solutions for asymptotically linear nonperiodic Hamiltonian elliptic system}, J. Math. Anal. Appl. 399 (2) (2013) 433-441. \bibitem{ZTZ} J. Zhang, X. H. Tang, W. Zhang; \emph{Ground-state solutions for superquadratic Hamiltonian elliptic systems with gradient terms}, Nonlinear Anal. 95 (2014), 1-10. \bibitem{ZTZ1} J. Zhang, X. H. Tang, W. Zhang; \emph{Semiclassical solutions for a class of Schr\"odinger system with magnetic potentials}, J. Math. Anal. Appl. 414 (1) (2014), 357-371. \bibitem{ZZ} L. G. Zhao, F. K. Zhao; \emph{On ground state solutions for superlinear Hamiltonian elliptic systems}, Z. Angew. Math. Phys. 64 (3) (2013), 403-418. \bibitem{ZZD1} F. K. Zhao, L. G. Zhao, Y. H. Ding; \emph{Multiple solutions for asymptotically linear elliptic systems}, NoDEA Nonlinear Differential Equations Appl. 15 (6) (2008), 673-688. \bibitem{ZZD2} F. K. Zhao, L. G. Zhao, Y. H. Ding; \emph{A note on superlinear Hamiltonian elliptic systems}, J . Math. Phys. 50 (11) (2009) 507-518. \bibitem{ZZD3} F. K. Zhao, L. G. Zhao, Y. H. Ding; \emph{Infinitely many solutions for asymptotically linear periodic Hamiltonian elliptic systems}, ESAIM Control Optim. Calc. Var. 16 (1) (2010) 77-91. \bibitem{ZZD4} F. K. Zhao, L. G. Zhao, Y. H. Ding; \emph{Multiple solutions for a superlinear and periodic elliptic system on $\mathbb{R}^N$}, Z. Angew. Math. Phys. 62 (3) (2011) 495-511. \end{thebibliography} \end{document}