\documentclass[reqno]{amsart} \usepackage{hyperref} \usepackage{graphicx} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2017 (2017), No. 54, pp. 1--11.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu} \thanks{\copyright 2017 Texas State University.} \vspace{8mm}} \begin{document} \title[\hfilneg EJDE-2017/54\hfil Existence of positive solutions] {Existence of positive solutions to perturbed nonlinear Dirichlet problems involving \\ critical growth} \author[H. Zhang, R. Zhang \hfil EJDE-2017/54\hfilneg] {Huixing Zhang, Ran Zhang} \address{Huixing Zhang (corresponding author)\newline Department of Mathematics, China University of Mining and Technology, Xuzhou, Jiangsu 221116, China} \email{huixingzhangcumt@163.com} \address{Ran Zhang \newline Department of Mathematics, China University of Mining and Technology, Xuzhou, Jiangsu 221116, China} \email{ranzhang\_zhangran@163.com} \dedicatory{Communicated by Zhaosheng Feng} \thanks{Submitted April 20, 2016. Published February 21, 2017.} \subjclass[2010]{35B33, 35J60, 35J65} \keywords{Perturbed nonlinear Dirichlet problem; critical growth; \hfill\break\indent Palais-Smale condition; variational methods} \begin{abstract} We consider the following perturbed nonlinear elliptic problem with critical growth \begin{gather*} -\varepsilon^2\Delta u+V(x)u=f(x)|u|^{p-2}u +\frac{\alpha}{\alpha+\beta}K(x)|u|^{\alpha-2}u|v|^\beta,\quad x\in \mathbb{R}^N,\\ -\varepsilon^2\Delta v+V(x)v=g(x)|v|^{p-2}v +\frac{\beta}{\alpha+\beta}K(x)|u|^\alpha|v|^{\beta-2}v,\quad x\in \mathbb{R}^N,\\ u(x),\quad v(x)\to 0 \quad \text{as } |x|\to\infty. \end{gather*} Using variational methods, we prove the existence of positive solutions. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{definition}[theorem]{Definition} \allowdisplaybreaks \section{Introduction} In this article, we discuss the following perturbed elliptic system involving critical growth \begin{equation} \label{eq1.1} \begin{gathered} -\varepsilon^2\Delta u+V(x)u=f(x)|u|^{p-2}u+\frac{\alpha}{\alpha+\beta}K(x) |u|^{\alpha-2}u|v|^\beta,\quad x\in \mathbb{R}^N,\\ -\varepsilon^2\Delta v+V(x)v=g(x)|v|^{p-2}v+\frac{\beta}{\alpha+\beta}K(x) |u|^\alpha|v|^{\beta-2}v,\quad x\in \mathbb{R}^N,\\ u(x),\quad v(x)\to 0 \quad \text{as } |x|\to\infty, \end{gathered} \end{equation} where $2< p<2^{\ast}$, $\alpha>1,\beta>1$ satisfy $\alpha+\beta=2^{\ast}$, $2^{\ast}=2N/(N-2)(N\geq3)$ is the critical Sobolev exponent. We assume that $V(x),K(x),f(x)$ and $g(x)$ satisfy the following conditions: \begin{itemize} \item[(H1)] $V\in C(\mathbb{R}^N,\mathbb{R})$, $V(0)=\inf_{x\in\mathbb{R}^N} V(x)=0$ and there exists $b>0$ such that the set $\nu^{b}:=\{x\in \mathbb{R}^{N}:V(x)0$, $\alpha,\beta>1$ satisfying $\alpha+\beta=2^{\ast}$. Many papers were devoted to the existence results of elliptic boundary valued problems on bounded domain with the Sobolev critical exponents. However, to the known of our knowledge, no studies were conducted on the existence of semi-classical solutions to \eqref{eq1.1} in $\mathbb{R}^N$. In this paper, we study system \eqref{eq1.1} in the whole space involving the critical growth. The main difficulty of this problem is the lack of compactness of the Sobolev embedding. To overcome this difficulty, we follow the approach originally developed in \cite{Squassina}. Namely, we will show that the corresponding energy functional of problem \eqref{eq1.1} satisfies the compactness condition at the levels less than some certain constant $c$. Our result complements the study made in \cite{Hsu2,Liu,Wu} in the sense that, in those papers, only the subcritical growth or the problem on bounded domain were considered. Let $\lambda=\varepsilon^{-2}$. Then \eqref{eq1.1} can be rewritten as \begin{equation} \label{eq1.6} \begin{gathered} -\Delta u+\lambda V(x)u=\lambda f(x)|u|^{p-2}u +\frac{\lambda \alpha}{\alpha+\beta}K(x)|u|^{\alpha-2}u|v|^\beta,\quad x\in \mathbb{R}^N,\\ -\Delta v+\lambda V(x)v=\lambda g(x)|v|^{p-2}v +\frac{\lambda \beta}{\alpha+\beta}K(x)|u|^\alpha|v|^{\beta-2}v,\quad x\in \mathbb{R}^N,\\ u(x),\quad v(x)\to 0 \quad \text{as } |x|\to\infty. \end{gathered} \end{equation} Since \eqref{eq1.1} and \eqref{eq1.6} are equivalent, then we will focus on system \eqref{eq1.6}. \begin{theorem} \label{thm1} Assume {\rm (H1)--(H3)} hold. Then for any $\sigma>0$, there is $\Lambda_{\sigma}>0$ such that if $\lambda>\Lambda_{\sigma}$, problem \eqref{eq1.6} has at least one solution $(u_\lambda, v_\lambda)$ that satisfies \begin{equation} \label{eq1.7} \frac{p-2}{2p}\int_{\mathbb{R}^{N}}(|\nabla u_{\lambda}|^2 +|\nabla v_{\lambda}|^2+\lambda V(x)(|u_{\lambda}|^2 +|v_{\lambda}|^2))\leq \sigma\lambda^{1-\frac{N}{2}}. \end{equation} \end{theorem} This article is organized as follows. In section 2, we show the $(PS)_c$ condition holds for $I_\lambda$ with some level $c$. In section 3, we obtain that the functional associated to \eqref{eq1.2} possesses the mountain geometry structure. Section 4 is devoted to the proof of the main result. \section{Palais-Smale condition} Let $E=E_{\lambda} \times E_{\lambda}$ be the Hilbert space with norm $$ \|(u,v)\|_E=(\int_{\mathbb{R}^{N}}(|\nabla u|^2+ \lambda V(x)u^2+|\nabla v|^2+ \lambda V(x)v^2))^\frac{1}{2}, $$ for any $(u,v)\in E$. Meanwhile, the space $$ E_{\lambda}=\{u\in H^1(\mathbb{R}^{N}):\int_{\mathbb{R}^{N}} \lambda V(x)u^2<\infty\ , \lambda>0\} $$ is a Hilbert space equipped with the inner product $$ (u,v)_{E_\lambda}=\int_{\mathbb{R}^{N}}(\nabla u\nabla v+\lambda V(x)uv). $$ We will show the existence of nontrivial solutions of \eqref{eq1.6} by searching for critical points of the functional associated to \eqref{eq1.6}, \begin{align*} I_{\lambda}(u,v) &=\frac{1}{2}\int_{\mathbb{R}^{N}}(|\nabla u|^2+\lambda V(x)u^2 +|\nabla v|^2+\lambda V(x)v^2)\\ &\quad -\frac{\lambda}{p}\int_{\mathbb{R}^{N}}(f(x)|u|^p +g(x)|v|^p)-\frac{\lambda}{\alpha+\beta} \int_{\mathbb{R}^{N}}K(x)|u|^{\alpha}|v|^{\beta}. \end{align*} In fact, the critical points of the functional $I_{\lambda}$ are the weak solutions of \eqref{eq1.6}. Recall that the weak solution $(u,v)$ of \eqref{eq1.6} satisfies \begin{align*} &\int_{\mathbb{R}^{N}}(\nabla u\nabla \varphi+\lambda V(x)u\varphi +\nabla v\nabla \psi+\lambda V(x)v\psi)\\ &=\lambda\int_{\mathbb{R}^{N}}(f(x)|u|^{p-2}u\varphi+g(x)|v|^{p-2}v\psi) +\frac{\lambda\alpha}{\alpha+\beta} \int_{\mathbb{R}^{N}}K(x) |u|^{\alpha-2}u|v|^{\beta}\varphi\\ &\quad +\frac{\lambda\beta}{\alpha+\beta} \int_{\mathbb{R}^{N}}K(x)|u|^{\alpha}|v|^{\beta-2}v\psi, \end{align*} for all $(\varphi,\psi)\in E$. Based on the assumptions of Theorem \ref{thm1}, we can show that $I_\lambda\in C^1(E,\mathbb{R})$ \cite{Mawhin}. \subsection*{Notation} $L^p(\mathbb{R}^{N})$, $1 \leq p< \infty$, denotes the Lebesgue spaces with $\|\cdot\|_p$. The dual space of a Banach space $E$ will be denoted by $E^{\ast}$. $B_r:=\{x\in \mathbb{R}^N:|x|\leq r\}$ is the ball in $\mathbb{R}^{N}$. $c,c_i$ represent various positive constants. Let $C_0^{\infty}(\mathbb{R}^{N})$ denote the collection of smooth functions with compact support. $o(1)$ denotes $o(1)\to 0$ as $n\to\infty$. Let $S_{\alpha,\beta}$ be the best Sobolev embedding constant defined by \begin{equation} S_{\alpha,\beta}= \inf_{u,v \in H^{1}(\mathbb{R}^{N})} \frac{\int_{\mathbb{R}^{N}}(|\nabla u|^2+|\nabla v|^2)} {\big(\int_{\mathbb{R}^{N}}|u|^{\alpha}|v|^{\beta}\big)^{\frac{2}{\alpha+\beta}}}. \end{equation} We have \[ S_{\alpha,\beta}= \Big((\frac{\alpha}{\beta})^{\frac{\beta}{\alpha+\beta}} +(\frac{\beta}{\alpha})^{\frac{\alpha}{\alpha+\beta}}\Big)S, \] where $S$ is the best Sobolev embedding constant defined by \[ S= \inf_{u \in H^{1}(\mathbb{R}^{N})} \frac{\int_{\mathbb{R}^{N}}|\nabla u|^2} {\big(\int_{\mathbb{R}^{N}}|u|^{2^{*}}\big)^{2/2^*}}. \] Next, we will find the range of $c$ where the $(PS)_c$ condition holds for the functional $I_{\lambda}$. \begin{definition} \label{def2.1}\rm Let $I\in C^{1}(E,\mathbb{R})$. \begin{itemize} \item[(1)] A sequence $\{z_n\}\subset E$ is called a $(PS)_c$ sequence in $E$ for $I$ if $I(z_n)=c+o(1)$ and $I'(z_n)=o(1)$ strongly in $E^{\ast}$ as $n\to\infty$. \item[(2)] $I$ satisfies $(PS)_c$ condition if any $(PS)_c$ sequence $\{z_n\}$ in $E$ for $I$ has a convergent subsequence. \end{itemize} \end{definition} \begin{lemma} \label{lem2.2} If the sequence $\{(u_n,v_n)\}\subset E$ is a $(PS)_c$ sequence for $I_\lambda$, then we have $c\geq 0$ and $\{(u_n,v_n)\}$ is bounded in the space $E$. \end{lemma} \begin{proof} We have \begin{align*} &I_{\lambda}(u_n,v_n)-\frac{1}{p}I_{\lambda}'(u_n,v_n)(u_n,v_n)\\ &= \frac{1}{2}\|(u_n,v_n)\|_{E}^2-\frac{\lambda}{p}\int_{\mathbb{R}^{N}} (f(x)|u_n|^p+g(x)|v_n|^p)-\frac{\lambda}{\alpha+\beta} \int_{\mathbb{R}^{N}}K(x)|u_n|^{\alpha}|v_n|^{\beta}\\ &\quad -\frac{1}{p}[\|(u_n,v_n)\|_{E}^2-\lambda\int_{\mathbb{R}^{N}} (f(x)|u_n|^p+g(x)|v_n|^p) -\lambda\int_{\mathbb{R}^{N}}K(x)|u_n|^{\alpha}|v_n|^{\beta}]\\ &= (\frac{1}{2}-\frac{1}{p})\|(u_n,v_n)\|_{E}^2 +(\frac{1}{p}-\frac{1}{\alpha+\beta})\lambda \int_{\mathbb{R}^{N}}K(x)|u_n|^{\alpha}|v_n|^{\beta}. \end{align*} From this and $20$, there is $r_\varepsilon>0$ with $r\geq r_\varepsilon$ $$ \limsup_{j\to\infty} \int_{B_j\setminus B_r}(|u_{n_j}|^d+|v_{n_j}|^d)\leq \varepsilon $$ where $2\leq d<2^{\ast}$. \end{lemma} \begin{proof} By Lemma \ref{lem2.2}, the $(PS)_c$ sequence $\{(u_n,v_n)\}$ for $I_\lambda$ is bounded in $E$. So, we assume $(u_n,v_n)\rightharpoonup (u,v)$ in $E$, $u_n\to u$, $v_n\to v$ a.e. in $\mathbb{R}^{N}$ and $(u_n,v_n)\to (u,v)$ in $L_{\rm loc}^{d}(\mathbb{R}^{N})\times L_{\rm loc}^{d}(\mathbb{R}^{N})$ for any $2\leq d<2^{\ast}$. For each $j\in \mathbb{N}$, we have \begin{align*} \int_{B_j}(|u_{n}|^d+|v_{n}|^d)\to \int_{B_j}(|u|^d+|v|^d). \end{align*} Thus, there exists $n_0\in \mathbb{N}$ such that \begin{align*} \int_{B_j}(|u_{n}|^d+|v_{n}|^d-|u|^d-|v|^d)<\frac{1}{j}\,, \end{align*} for all $n\geq n_0+1$. Without loss of generality, we choose $n_j=n_0+j$ such that \[ \int_{B_j}(|u_{n_j}|^d+|v_{n_j}|^d-|u|^d-|v|^d)<\frac{1}{j}\,. \] It is easy to show that there is a $r_\varepsilon$ satisfying \[ \int_{\mathbb{R}^{N}\setminus B_r}(|u|^d+|v|^d)<\varepsilon \quad \text{for all } r \geq r_\varepsilon. \] Since \begin{align*} &\int_{B_j\setminus B_r}(|u_{n_j}|^d+|v_{n_j}|^d)\\ &< \frac{1}{j}+\int_{\mathbb{R}^{N}\setminus B_r}(|u|^d+|v|^d) +\int_{B_r}(|u|^d-|u_{n_j}|^d+|v|^d-|v_{n_j}|^d). \end{align*} In connection with $(u_n,v_n)\to (u,v)$ in $L_{\rm loc}^{d}(\mathbb{R}^{N}) \times L_{\rm loc}^{d}(\mathbb{R}^{N})$, the lemma follows. \end{proof} Let $\eta\in C^{\infty}(\mathbb{R}^+,[0,1])$ be a smooth function satisfying $\eta(t)=1$ if $t\leq 1$ and $\eta(t)=0$ if $t\geq 2$. Define $\tilde{u}_j(x)=\eta(2|x|/j)u(x)$ and $\tilde{v}_j(x)=\eta(2|x|/j)v(x)$, then \begin{equation} \label{eq2.2} (\tilde{u}_j,\tilde{v}_j)\to (u,v) \quad \text{in } E \text{ as } j\to\infty. \end{equation} \begin{lemma} \label{lem2.4} One has \begin{gather*} \lim_{j\to\infty}|\int_{\mathbb{R}^{N}}f(x)(|u_{n_j}|^{p-2}u_{n_j} -|u_{n_j}-\tilde{u}_j|^{p-2}(u_{n_j}-\tilde{u}_j) -|\tilde{u}_j|^{p-2}\tilde{u}_j)\varphi|=0, \\ \lim_{j\to\infty}|\int_{\mathbb{R}^{N}}g(x)(|v_{n_j}|^{p-2}v_{n_j} -|v_{n_j}-\tilde{v}_j|^{p-2}(v_{n_j}-\tilde{v}_j) -|\tilde{v}_j|^{p-2}\tilde{v}_j)\psi|=0 \end{gather*} uniformly in $(\varphi,\psi)\in E$ with $\|(\varphi,\psi)\|_E\leq 1$. \end{lemma} The proof of the above lemma is similar to the one \cite[Lemma 3.4]{Ding}, so we omit it. \begin{lemma} \label{lem2.5} Passing to a subsequence, we have \begin{gather*} I_{\lambda}(u_n-\tilde{u}_n,v_n-\tilde{v}_n)\to c-I_{\lambda}(u,v),\\ I_{\lambda}'(u_n-\tilde{u}_n,v_n-\tilde{v}_n)\to 0\ \text{in}\ E^{\ast}. \end{gather*} \end{lemma} \begin{proof} From $(u_n,v_n)\rightharpoonup (u,v)$ and $(\tilde{u}_n,\tilde{v}_n)\to (u,v)$ in $E$, we obtain \begin{align*} &I_{\lambda}(u_n-\tilde{u}_n,v_n-\tilde{v}_n)\\ &= I_{\lambda}(u_n,v_n)-I_{\lambda}(\tilde{u}_n,\tilde{v}_n)\\ &\quad +\frac{\lambda}{p^{\ast}}\int_{\mathbb{R}^{N}}K(x) (|u_n|^{\alpha}|v_n|^{\beta}-|u_n-\tilde{u}_n|^{\alpha} |v_n-\tilde{v}_n|^{\beta}-|\tilde{u}_n|^{\alpha}|\tilde{v}_n|^{\beta})\\ &\quad +\frac{\lambda}{p}\int_{\mathbb{R}^{N}}f(x)(|u_n|^p -|u_n-\tilde{u}_n|^p-|\tilde{u}_n|^p)\\ &\quad +\frac{\lambda}{p}\int_{\mathbb{R}^{N}}g(x)(|v_n|^p -|v_n-\tilde{v}_n|^p-|\tilde{v}_n|^p)+o(1). \end{align*} Similar to the proof of Br\'{e}zis-Lieb Lemma \cite{Brezis}, it is easy to obtain \begin{gather*} \lim_{n\to\infty}\int_{\mathbb{R}^{N}}K(x)(|u_n|^{\alpha}|v_n|^{\beta} -|u_n-\tilde{u}_n|^{\alpha}|v_n-\tilde{v}_n|^{\beta} -|\tilde{u}_n|^{\alpha}|\tilde{v}_n|^{\beta})=0, \\ \lim_{n\to\infty}\int_{\mathbb{R}^{N}}f(x)(|u_n|^p-|u_n-\tilde{u}_n|^p -|\tilde{u}_n|^p)=0, \\ \lim_{n\to\infty}\int_{\mathbb{R}^{N}}g(x)(|v_n|^p-|v_n-\tilde{v}_n|^p -|\tilde{v}_n|^p)=0. \end{gather*} Observing that $I_{\lambda}(u_n,v_n)\to c$ and $I_{\lambda}(\tilde{u}_n,\tilde{v}_n)\to I_{\lambda}(u,v)$, we have $$ I_{\lambda}(u_n-\tilde{u}_n,v_n-\tilde{v}_n)\to c-I_{\lambda}(u,v). $$ In addition, for any $(\varphi,\psi)\in E$, we obtain \begin{align*} &I_{\lambda}'(u_n-\tilde{u}_n,v_n-\tilde{v}_n)(\varphi,\psi)\\ &= I_{\lambda}'(u_n,v_n)(\varphi,\psi)-I_{\lambda}' (\tilde{u}_n,\tilde{v}_n)(\varphi,\psi) +\frac{\lambda\alpha}{\alpha+\beta}\int_{\mathbb{R}^{N}}K(x) \Big(|u_n|^{\alpha-2}u_n|v_n|^{\beta} \\ &\quad -|u_n-\tilde{u}_n|^{\alpha-2} (u_n-\tilde{u}_n)|v_n-\tilde{v}_n|^{\beta}-|\tilde{u}_n|^{\alpha-2} \tilde{u}_n|\tilde{v}_n|^{\beta}\Big)\varphi\\ &\quad +\frac{\lambda\beta}{\alpha+\beta}\int_{\mathbb{R}^{N}}K(x) \Big(|u_n|^{\alpha}|v_n|^{\beta-2}v_n-|u_n-\tilde{u}_n|^{\alpha}|v_n -\tilde{v}_n|^{\beta-2}(v_n-\tilde{v}_n) \\ &\quad -|\tilde{u}_n|^{\alpha} |\tilde{v}_n|^{\beta-2}\tilde{v}_n\Big)\psi\\ &\quad +\lambda\int_{\mathbb{R}^{N}}f(x) \big(|u_n|^{p-2}u_n-|u_n-\tilde{u}_n|^{p-2}(u_n-\tilde{u}_n) -|\tilde{u}_n|^{p-2}\tilde{u}_n\big)\varphi\\ &\quad +\lambda\int_{\mathbb{R}^{N}}g(x) \big(|v_n|^{p-2}v_n-|v_n-\tilde{v}_n|^{p-2}(v_n-\tilde{v}_n) -|\tilde{v}_n|^{p-2}\tilde{v}_n\big)\psi. \end{align*} It is standard to check that \begin{align*} &\lim_{n\to\infty}\int_{\mathbb{R}^{N}}K(x)(|u_n|^{\alpha-2}u_n|v_n|^{\beta} -|u_n-\tilde{u}_n|^{\alpha-2}(u_n-\tilde{u}_n)|v_n-\tilde{v}_n|^{\beta} \\ & -|\tilde{u}_n|^{\alpha-2}\tilde{u}_n|\tilde{v}_n|^{\beta})\varphi=0, \\ &\lim_{n\to\infty}\int_{\mathbb{R}^{N}}K(x)(|u_n|^{\alpha}|v_n|^{\beta-2}v_n -|u_n-\tilde{u}_n|^{\alpha}|v_n-\tilde{v}_n|^{\beta-2}(v_n-\tilde{v}_n)\\ & -|\tilde{u}_n|^{\alpha}|\tilde{v}_n|^{\beta-2}\tilde{v}_n)\psi=0 \end{align*} uniformly in $\|(\varphi,\psi)\|_E\leq 1$. By Lemma \ref{lem2.4} and $I_{\lambda}'(u_n,v_n)\to 0$, we complete the proof. \end{proof} Set $u_n^1=u_n-\tilde{u}_n$ and $v_n^1=v_n-\tilde{v}_n$, then $u_n-u=u_n^1+(\tilde{u}_n-u)$ and $v_n-v=v_n^1+(\tilde{v}_n-v)$. Then $(u_n,v_n)\to (u,v)$ in $E$ if and only if $(u_n^1,v_n^1)\to (0,0)$ in $E$. Observe that \begin{align*} I_{\lambda}(u_n^1,v_n^1)-\frac{1}{2}I_{\lambda}'(u_n^1,v_n^1)(u_n^1,v_n^1) &= \big(\frac{1}{2}-\frac{1}{\alpha+\beta}\big)\lambda \int_{\mathbb{R}^{N}}K(x)|u_n^1|^{\alpha}|v_n^1|^{\beta}\\ &\quad +(\frac{1}{2}-\frac{1}{p})\lambda \int_{\mathbb{R}^{N}}(f(x)|u_n^1|^p+g(x)|v_n^1|^p)\\ &\geq \frac{\lambda}{N}K_0\int_{\mathbb{R}^{N}}|u_n^1|^{\alpha}|v_n^1|^{\beta}, \end{align*} where $K_0=\inf_{x\in \mathbb{R}^{N}}K(x)>0$. In connection with $I_{\lambda}(u_n^1,v_n^1)\to c-I_{\lambda}(u,v)$ and $I_{\lambda}'(u_n^1,v_n^1)\to 0\ \text{in}\ E^{\ast}$, we obtain \begin{equation} \label{eq2.3} \int_{\mathbb{R}^{N}}|u_n^1|^{\alpha}|v_n^1|^{\beta} \leq\frac{N(c-I_\lambda(u,v))}{\lambda K_0}+o(1). \end{equation} In addition, by (H2) and (H3), for any $b>0$, there is a constant $C_b>0$ such that \begin{align*} &\int_{\mathbb{R}^{N}}(K(x)|u_n^1|^{\alpha}|v_n^1|^{\beta} +f(x)|u_n^1|^p+g(x)|v_n^1|^p)\\ &\leq b(\|u_n^1\|_2^2+\|v_n^1\|_2^2) +C_b\int_{\mathbb{R}^{N}}|u_n^1|^{\alpha}|v_n^1|^{\beta}. \end{align*} Let $V_b(x):=\max\{V(x),b\}$, where $b$ is the positive constant in the assumption $(H_1)$. Since the set $\nu^{b}:=\{x\in \mathbb{R}^{N}:V(x)0$ (independent of $\lambda$) such that the functional $I_{\lambda}(u,v)$ satisfies the $(PS)_c$ condition for all $c< \alpha_0\lambda^{1-\frac{N}{2}}$. \end{lemma} \begin{proof} We can check that, for any $(PS)_c$ sequence $\{(u_n,v_n)\}\subset E$ with $(u_n,v_n)\rightharpoonup (u,v)$, either $(u_n,v_n)\to (u,v)$ or $c-I_{\lambda}(u,v)\geq \alpha_0\lambda^{1-\frac{N}{2}}$. On the contrary, suppose that $(u_n,v_n) \not\rightarrow(u,v)$; then $$ \lim\inf_{n\to\infty}\|(u_n,v_n)\|_E>0 $$ and $c-I_{\lambda}(u,v)>0$. Based on the above conclusions, we easily get that the functional $I_\lambda(u,v)$ satisfies the $(PS)_c$ condition for all $c<\alpha_0\lambda^{1-\frac{N}{2}}$. \end{proof} \section{Mountain-pass structure} We consider $\lambda\geq1$ and check that the functional $I_\lambda$ possesses the mountain-pass structure. \begin{lemma} \label{lem3.1} Assume that {\rm (H1)--(H3)} are satisfied. Then there exists $\alpha_\lambda,\ \rho_\lambda>0$ such that $$ I_{\lambda}(u,v)>0\ \text{if}\ 0<\|(u,v)\|_E<\rho_\lambda,\quad I_{\lambda}(u,v)\geq\alpha_\lambda\ \text{if}\ \|(u,v)\|_E=\rho_\lambda. $$ \end{lemma} \begin{proof} Note that \begin{align*} I_{\lambda}(u,v) &= \frac{1}{2}\int_{\mathbb{R}^{N}}(|\nabla u|^2+\lambda V(x)u^2 +|\nabla v|^2+\lambda V(x)v^2)\\ &\quad -\frac{\lambda}{p}\int_{\mathbb{R}^{N}}(f(x)|u|^p+g(x)|v|^p) -\frac{\lambda}{\alpha+\beta} \int_{\mathbb{R}^{N}}K(x)|u|^{\alpha}|v|^{\beta}. \end{align*} It is clear that, for each $s\in[2,2^{\ast}]$, there is $c_s$ such that if $\lambda \geq 1$, $$ \|u\|_s\leq c_s\|u\|_{E_\lambda} \quad \text{for all } u\in E_\lambda. $$ By Young inequality, we have \[ |u|^{\alpha}|v|^{\beta}\leq\frac{\alpha}{\alpha+\beta}|u|^{\alpha+\beta} +\frac{\beta}{\alpha+\beta}|v|^{\alpha+\beta}. \] Furthermore, we obtain \begin{equation} \label{eq3.1} \int_{\mathbb{R}^{N}}K(x)|u|^{\alpha}|v|^{\beta} \leq c_1(\|u\|_{2^{\ast}}^{2^{\ast}}+\|v\|_{2^{\ast}}^{2^{\ast}}) \leq c_1c_{2^{\ast}}\|(u,v)\|_{E}^{2^{\ast}}. \end{equation} Combining (H3) and \eqref{eq3.1}, there is a constant $c_\delta$ such that \[ I_{\lambda}(u,v)\geq \frac{1}{4}\|(u,v)\|_{E}^2-c_\delta\|(u,v)\|_E^{2^{\ast}} =\frac{1}{4}\|(u,v)\|_{E}^2(1-4c_\delta\|(u,v)\|_{E}^{2^{\ast}-2}). \] Set $\rho_\lambda=(\frac{1}{8c_\delta})^{\frac{1}{2^{\ast}-2}}$, it implies $$ I_{\lambda}(u,v)\geq \frac{1}{8}\rho_\lambda^2=: \alpha_\lambda>0\quad \text{if } \|(u,v)\|_E=\rho_\lambda. $$ The proof is complete. \end{proof} \begin{lemma} \label{lem3.2} For any finite dimensional subspace $F\subset E$, we have $$ I_{\lambda}(u,v)\to -\infty\quad \text{as } \|(u,v)\|_{E}\to\infty \quad \text{for } (u,v)\in F. $$ \end{lemma} \begin{proof} From assumptions (H2) and (H3), it follows that $$ I_{\lambda}(u,v)\leq\frac{1}{2}\|(u,v)\|_{E}^2-\lambda c_0\|(u,v)\|_{p}^p\quad \text{for all } (u,v)\in F. $$ In connection with the fact that all norms in a finite-dimensional space are equivalent and $p>2$, we easily get the desired conclusion. \end{proof} \begin{lemma} \label{lem3.3} For any $\sigma>0$, there is $\Lambda_\sigma>0$ such that for each $\lambda\geq \Lambda_\sigma$, there exists $\bar{e}_\lambda\in E$ with $\|\bar{e}_\lambda\|_E>\rho_\lambda$, we have $I_\lambda(\bar{e}_\lambda)\leq 0$ and $$ \max_{t\geq 0}I_{\lambda}(t\bar{e}_\lambda)\leq\sigma\lambda^{1-\frac{N}{2}}, $$ where $\rho_\lambda$ is defined in Lemma \ref{lem3.1}. \end{lemma} \begin{proof} Define the functionals \begin{gather*} \Phi_{\lambda}(u,v)=\frac{1}{2}\int_{\mathbb{R}^{N}}(|\nabla u|^2 +\lambda V(x)|u|^2+|\nabla v|^2+\lambda V(x)|v|^2) -\lambda c_0\int_{\mathbb{R}^{N}}(|u|^p+|v|^p), \\ \Psi_{\lambda}(u,v)=\frac{1}{2}\int_{\mathbb{R}^{N}}(|\nabla u|^2 +|\nabla v|^2+V(\lambda^{-\frac{1}{2}}x)(|u|^2+|v|^2)) -c_0\int_{\mathbb{R}^{N}}(|u|^p+|v|^p). \end{gather*} We obtain that $\Phi_{\lambda}\in C^1(E)$ and $I_{\lambda}(u,v)\leq\Phi_{\lambda}(u,v)$ for all $(u,v)\in E$. Observe that $$ \inf\big\{\int_{\mathbb{R}^{N}}|\nabla \phi|^2: \phi\in C_0^{\infty}(\mathbb{R}^{N},\mathbb{R}),\|\phi\|_{p}=1\big\}=0. $$ For any $\delta>0$, there are $\phi_\delta, \psi_\delta\in C_0^{\infty}(\mathbb{R}^{N},\mathbb{R})$ with $\|\phi_\delta\|_{p}=\|\psi_\delta\|_{p}=1$ such that $$ \operatorname{supp}(\phi_\delta,\psi_\delta) \subset B_{r_\delta}(0)\quad \text{and}\quad \|\nabla\phi_\delta\|_2^2,\|\nabla\psi_\delta\|_2^2<\delta. $$ Let $e_{\lambda}(x)=(\phi_\delta(\sqrt{\lambda}x), \psi_\delta(\sqrt{\lambda}x))$, then $\operatorname{supp}e_{\lambda}\subset B_{\lambda^{-\frac{1}{2}}r_{\delta}}(0)$. Furthermore, $$ \Phi_{\lambda}(te_{\lambda})=\lambda^{1-\frac{N}{2}} \Psi_{\lambda}(t\phi_\delta,t\psi_\delta). $$ It is clear that \begin{align*} \max_{t\geq 0}\Psi_{\lambda}(t\phi_\delta,t\psi_\delta) &\leq \frac{p-2}{2p(p c_0)^{\frac{2}{p-2}}} \Big\{\int_{\mathbb{R}^{N}}(|\nabla\phi_\delta|^2 +V(\lambda^{-\frac{1}{2}}x)|\phi_\delta|^2)\Big\}^{\frac{p}{p-2}}\\ &\quad +\frac{p-2}{2p(p c_0)^{\frac{2}{p-2}}} \Big\{\int_{\mathbb{R}^{N}}(|\nabla\psi_\delta|^2+V(\lambda^{-\frac{1}{2}}x) |\psi_\delta|^2)\Big\}^{\frac{p}{p-2}}. \end{align*} Combining $V(0)=0$ and $\operatorname{supp}(\phi_\delta,\psi_\delta)\subset B_{r_\delta}(0)$, there is $\Lambda_\delta>0$ such that for all $\lambda\geq \Lambda_\delta$, we have \begin{align*} \max_{t\geq 0}\Phi_{\lambda}(t\phi_\delta,t\psi_\delta) \leq \lambda^{1-\frac{N}{2}}\frac{(p-2)}{p(p c_0)^{\frac{2}{p-2}}} (2\delta)^{\frac{p}{p-2}}. \end{align*} Thus, for all $\lambda\geq \Lambda_\delta$, \begin{equation} \label{eq3.2} \max_{t\geq 0}I_{\lambda}(te_\lambda) \leq \lambda^{1-\frac{N}{2}}\frac{(p-2)}{p(p c_0)^{\frac{2}{p-2}}}(2\delta)^{\frac{p}{p-2}}. \end{equation} For any $\sigma>0$, we can choose $\delta>0$ enough small such that $$ \frac{(p-2)}{p(p c_0)^{\frac{2}{p-2}}}(2\delta)^{\frac{p}{p-2}}\leq \sigma $$ and $e_\lambda(x)=(\phi_\delta(\sqrt{\lambda}x),\psi_\delta(\sqrt{\lambda}x))$. Taking $\Lambda_\delta=\Lambda_\sigma$, there is $\bar{t}_\lambda>0$ such that $\|\bar{t}_\lambda e_\lambda\|_E>\rho_\lambda$ and $I_\lambda(te_\lambda)\leq0$ for all $t\geq \bar{t}_\lambda$. By \eqref{eq3.2}, $\bar{e}_\lambda=\bar{t}_\lambda e_\lambda$ satisfies the requirements. \end{proof} \section{Proof of Theorem \ref{thm1}} Define $$ c_\lambda=\inf_{\gamma\in \Gamma_\lambda}\max_{t\in[0,1]}I_\lambda(\gamma(t)), $$ where $\Gamma_\lambda=\{\gamma\in C([0,1],E):\gamma(0)=0,\gamma(1)=\bar{e}_\lambda\}$. In addition, for any $\sigma>0$ with $\sigma<\alpha_0$, there is $\Lambda_\sigma>0$ such that $\lambda\geq \Lambda_\sigma$. We can take $c_\lambda$ satisfying $c_\lambda\leq\sigma\lambda^{1-\frac{N}{2}}$. From the above results, the functional $I_\lambda$ satisfies $(PS)_{c_\lambda}$ condition and has the mountain-pass structure if $c_\lambda\leq\sigma\lambda^{1-\frac{N}{2}}$. Hence, there is $(u_\lambda,v_\lambda)\in E$ such that $$ I_\lambda(u_\lambda,v_\lambda)=c_\lambda \quad \text{and}\quad I_\lambda'(u_\lambda,v_\lambda)=0. $$ That is to say, $(u_\lambda,v_\lambda)$ is a weak solution of \eqref{eq1.6}. Similar to the arguments in \cite{Ding}, we also obtain that $(u_\lambda,v_\lambda)$ is a positive least energy solution. Furthermore, \begin{align*} I_\lambda(u_\lambda,v_\lambda) &= I_\lambda(u_\lambda,v_\lambda) -\frac{1}{p}I_\lambda'(u_\lambda,v_\lambda)(u_\lambda,v_\lambda)\\ &\geq (\frac{1}{2}-\frac{1}{p})\|(u_\lambda,v_\lambda)\|_{E}^2. \end{align*} This shows that $$ \frac{p-2}{2p}\|(u_\lambda,v_\lambda)\|_E^2 \leq I_\lambda(u_\lambda,v_\lambda)=c_\lambda\leq \sigma\lambda^{1-\frac{N}{2}}. $$ The proof is complete. \subsection*{Acknowledgements} The authors would like to thank the referees for their comments and suggestions about the original manuscript. This research was supported by the Fundamental Research Funds for the Central Universities (2015XKMS072). \begin{thebibliography}{00} \bibitem{Ahammou} Ahammou, A; \emph{Positive radial solutions of nonlinear elliptic system}. New York J. Math., \textbf{7} (2001), 267-280. \bibitem{Alves2} Alves, C. O.; de Morais Filho, D. C.; Souto, M. A. S.; \emph{On system of elliptic equations involving subcritical or critical Sobolev exponents}. Non. Anal., \textbf{42} (2000), 771-787. \bibitem{Ambrosetti} Ambrosetti, A.; Br\'{e}zis, H.; Cerami, G; \emph{Combined effects of concave and convex nonlinearities in some elliptic problems}. J. Funct. Anal., \textbf{122} (1994), 519-543. \bibitem{Benci} Benci, V.; Cerami, G; \emph{Existence of positive solutions of the equation $-\Delta u+a(x)u=u^{\frac{N+2}{N-2}}$ in $\mathbb{R}^{N}$}. J. Funct. Anal., \textbf{88} (1990), 90-117. \bibitem{Bozhkov} Bozhkov, Y.; Mitidieri, E.; \emph{Existence of multiple solutions for quasilinear systems via fibering method}. J. Diff. Equ., \textbf{190} (2003), 239-267. \bibitem{Brezis} Br\'{e}zis, H.; Lieb, E.; \emph{A relation between pointwise convergence of functions and convergence of functional}. Proc. Amer. Math. Soc., \textbf{88} (1983), 486-490. \bibitem{Brezis2} Br\'{e}zis, H.; Nirenberg, L.; \emph{Some variational problems with lack of compactness}. Proc. Symp. Pure Math., \textbf{45} (1986), 165-201. \bibitem{Cingolani} Cingolani, S.; Lazzo, M.; \emph{Multiple positive solutions to nonlinear Schr\"{o}dinger equation with competing potential functions}. J. Diff. Equ. \textbf{160} (2000), 118-138. \bibitem{Clapp} Clapp, M.; Ding, Y. H.; \emph{Minimal nodal solutions of a Schr\"{o}dinger equation with critical nonlinearity and symmetric potential}. Diff. Integ. Equ., \textbf{16} (2003), 981-992. \bibitem{de Figueiredo} de Figueiredo, D. G.; Felmer, P.; \emph{On superquadratic elliptic systems}. Trans. Amer. Math. Soc., \textbf{343} (1994), 99-116. \bibitem{Del} Del Pino, M.; Felmer, P.; \emph{Semi-classical states for nonlinear Schr\"{o}dinger equations}. Annales Inst. H. Poincar\'{e}, \textbf{15} (1998), 127-149. \bibitem{Ding} Ding, Y. H.; Lin, F. H.; \emph{Solutions of perturbed Schr\"{o}dinger equations with critical nonlinearity}. Calc. Var., \textbf{30} (2007), 231-249. \bibitem{Floer} Floer, A.; Weinstein, A.; \emph{Nonspreading wave packets for the cubic Schr\"{o}dinger equation with a bounded potential}. J. Funct. Anal., \textbf{69} (1986), 397-408. \bibitem{Han1} Han, P. G.; \emph{High energy positive solutions for a critical growth Dirichlet problem in noncontractible domains}. Non. Anal., \textbf{60} (2005), 369-387. \bibitem{Han2} Han, P. G.; \emph{The effect of the domain topology on the number of positive solutions of some elliptic systems involving critical Sobolev exponents}. Houston. J. Math., \textbf{32} (2006), 1241-1257. \bibitem{Hsu} Hsu, T. S. ; \emph{Multiple positive solutions for a critical quasilinear elliptic system with concave-convex nonlinearities}. Non. Anal., \textbf{71} (2009), 2688-2698. \bibitem{Hsu2} Hsu, T. S., Lin; H. L.; \emph{Multiple positive solutions for a critical elliptic system with concave-convex nonlinearities}. Proc. Roy. Soc. Edinburgh Sect. A., \textbf{139} (2009), 1163-1177. \bibitem{Liu} Liu, Z. X.; Han, P. G.; \emph{Infinitely many solutions for elliptic systems with critical exponents}. J. Math. Anal. Appl., \textbf{353} (2009), 544-552. \bibitem{Mawhin} Mawhin, J.; Willem, M; \emph{Critical Point Theory and Hamiltonian Systems}. Springer-Verlag, New York, 1989. \bibitem{Oh1990} Oh, Y. G.; \emph{On positive multi-lump bound states of nonlinear Schr\"{o}dinger equations under multiple well potential}. Comm. Math. Phys. \textbf{131} (1990), 223-253. \bibitem{Squassina} Squassina, M.; \emph{An eigenvalue problem for elliptic systems}. New York J. Math., \textbf{6} (2000), 101-123. \bibitem{Velin} Velin, J.; \emph{Existence results for some nonlinear elliptic system with lack of compactness}. Non. Anal. \textbf{52} (2003), 1017-1034. \bibitem{Wu} Wu, T. F.; \emph{The Nehari maniford for a semilinear elliptic system involving sign-changing weight functions}. Non. Anal. \textbf{68} (2008), 1733-1745. \end{thebibliography} \end{document}