\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2014 (2014), No. 251, pp. 1--11.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2014 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2014/251\hfil Semiclassical solutions] {Semiclassical solutions for linearly coupled Schr\"odinger equations} \author[S. Chen, X. H. Tang \hfil EJDE-2014/251\hfilneg] {Sitong Chen, Xianhua Tang} % in alphabetical order \address{Sitong Chen \newline School of Mathematics and Statistics, Central South University, Changsha, Hunan 410083, China} \email{sitongchen2041@hotmail.com} \address{Xianhua Tang \newline School of Mathematics and Statistics, Central South University, Changsha, Hunan 410083, China} \email{tangxh@mail.csu.edu.cn} \thanks{Submitted October 23, 2014. Published December 1, 2014.} \subjclass[2000]{35J20, 58E50} \keywords{Nonlinear Schr\"odinger equation; semiclassical solution; \hfill\break\indent coupled system} \begin{abstract} We consider the system of coupled nonlinear Schr\"odinger equations \begin{gather*} -\varepsilon^2\Delta u+a(x) u=H_{u}(x, u, v)+\mu(x) v, \quad x\in \mathbb{R}^N,\\ -\varepsilon^2\Delta v+b(x) v=H_{v}(x, u, v)+\mu(x) u, \quad x\in \mathbb{R}^N,\\ u,v\in H^1(\mathbb{R}^N), \end{gather*} where $N\ge 3$, $a, b, \mu \in C(\mathbb{R}^N)$ and $H_{u}, H_{v}\in C(\mathbb{R}^N\times \mathbb{R}^2, \mathbb{R})$. Under conditions that $a_0=\inf a=0$ or $b_0=\inf b=0$ and $|\mu(x)|^2\le \theta a(x)b(x)$ with $\theta\in(0, 1)$ and some mild assumptions on $H$, we show that the system has at least one nontrivial solution provided that $0<\varepsilon\le \varepsilon_0$, where the bound $\varepsilon_0$ is formulated in terms of $N, a, b$ and $H$. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{remark}[theorem]{Remark} \newtheorem{example}[theorem]{Example} \allowdisplaybreaks \newcommand{\R}{\mathbb{R}} \newcommand{\Z}{\mathbb{Z}} \newcommand{\N}{\mathbb{N}} \section{Introduction} In this article, we study the existence of semiclassical solutions of the system of coupled nonlinear Schr\"odinger equations \begin{equation}\label{PB1} \begin{gathered} -\varepsilon^2\Delta u+a(x) u=H_{u}(x, u, v)+\mu(x) v, \quad x\in \mathbb{R}^N,\\ -\varepsilon^2\Delta v+b(x) v=H_{v}(x, u, v)+\mu(x) u, \quad x\in \mathbb{R}^N,\\ u,v\in H^1(\mathbb{R}^N), \end{gathered} \end{equation} where $z:=(u, v)\in \mathbb{R}^2$, $N\ge3$, $a, b, \mu \in C(\mathbb{R}^N,\mathbb{R})$ and $H, H_{u}, H_{v}\in C(\mathbb{R}^{N}\times\mathbb{R}^2,\mathbb{R})$. Systems of this type arise in nonlinear optics \cite{AA}. In the past several years, there are many papers about the semiclassical solutions of the nonlinear perturbed Schr\"odinger equation $$ -\varepsilon^2\Delta u+V(x)u=f(x, u), \quad u \in H^{1}(\mathbb{R}^N) $$ under various hypotheses on the potential and the nonlinearity (see \cite{ABC, AMN, AMN1, DL, DS, DW, HT, Li, Pom, Ta2}). However, by Kaminow \cite{Kam}, we know that single-mode optical fibers are not really ``single-mode", but actually bimodal due to the presence of birefringence. And recently, different authors focused their attention on coupled nonlinear Schr\"odinger systems (see \cite{ACR, ACR1, CZ, CZ1, CZ2, CZ3, LT1}) which describe physical phenomena (see, e.g., \cite{AA, AEKS, EKKS}). In a recent article, \cite{CZ}, Chen and Zou studied the system of nonlinear Schr\"odinger equations \begin{equation}\label{Y1} \begin{gathered} -\varepsilon^2\Delta u+a(x) u=f(u)+\mu v, \quad x\in \mathbb{R}^N,\\ -\varepsilon^2\Delta v+b(x) v=g(v)+\mu u, \quad x\in \mathbb{R}^N,\\ u, v>0 \quad \text{in } \mathbb{R}^N, \quad u,v\in H^1(\mathbb{R}^N), \end{gathered} \end{equation} where $N, a$ and $b$ are the same as in \eqref{PB1}. Under the assumptions \begin{itemize} \item[(i)] there exists a constant $a_0>0$ such that $a(x),\ b(x) \geq a_0$ and $0 \le\mu 0$ and all $\mu \in (0, \mu_1]$ for some $\mu_1\in (0, a_0)$. Obviously, if $a_0=0$, their arguments become invalid due to the fact that $0 \leq \mu < a_0$ can not be satisfied. To the best of our knowledge, the existence of semiclassical solutions to system \eqref{PB1}, under the assumption of $a_0=\inf a=0$ or $b_0=\inf b=0$, has not ever been studied by variational methods. In addition, as the nonlinearity is non-autonomous and dependent on $u$ and $v$, the problem will become more complex. Motivated by \cite{CZ, LT, Ta1, Ta3}, we shall choose the case $a_0=\inf a=0$ or $b_0=\inf b=0$ as the objective of the present paper. Before presenting the main results, we introduce the following assumptions. \begin{itemize} \item[(A0)] $a(x)\ge a(0)=0$, $b(x)\ge 0$ and there exist $a_{0}, b_{0} > 0$ such that the sets $\mathcal{A}_{a_{0}} := \{x \in \mathbb{R}^N: a (x) 0$ such that the sets $\mathcal{A}_{a_{0}} := \{x \in \mathbb{R}^N: a (x)0$ such that $$ |H(x, z)|\le C(|z|+|z|^{p}), \quad \forall (x, z)\in \mathbb{R}^N\times \mathbb{R}^{2}; $$ \item[(H2)] $H_{z}(x, z)\cdot z=o(|z|^2)$, as $|z|\to 0$, uniformly in $x\in \mathbb{R}^N$ ; \item[(H3)] $\lim_{|z|\to \infty}\frac{|H(x, z)|}{|z|^2}=\infty$ uniformly in $x\in \mathbb{R}^{N}$; \item[(H4)] there exist $c_0>0$, $T_0>0$ and $q\in (2, 2^*)$ such that $$ H(x, u, 0)\ge c_0|u|^q, \quad \forall x \in \mathbb{R}^{N},\; u \in [-T_0, T_0] $$ and \[ u^{-2}h^{4-N}\int_{|x|\le h}H(\lambda^{-1/2}x, u/h, 0) \,\mathrm{d}x\ge \frac{(N^2+2)\omega_N}{2N(1-2^{-N})^2}, \] for all $h\ge 1$, $\lambda\ge 1$, $u\ge hT_0$; here and in the sequel, $\omega_N=\mathrm{meas}(B_1(0))=2\pi^{N/2}/N\Gamma(N/2)$; \item[(H4')] there exist $c_0>0$, $T_0>0$ and $q\in (2, 2^*)$ such that $$ H(x, 0, v)\ge c_0|v|^q, \quad \forall x \in \mathbb{R}^{N},\; v \in [-T_0, T_0] $$ and \[ v^{-2}h^{4-N}\int_{|x|\le h}H(\lambda^{-1/2}x, 0, v/h)\,\mathrm{d}x \ge \frac{(N^2+2)\omega_N}{2N(1-2^{-N})^2}, \] for all $h\ge 1$, $\lambda\ge 1$, $v\ge hT_0$; \item[(H5)] $\mathcal{H}(x, z):=\frac{1}{2}H_{z}(x, z)\cdot z-H(x, z)\ge 0$ for all $(x, z)\in \mathbb{R}^{N}\times \mathbb{R}^{2}$, and there exist $c_1>0$ and $\kappa>\max\{1, N/2\}$ such that $$ \frac{H_{z}(x, z)\cdot z}{|z|^2}\ge \frac{(1-\theta)m_0}{3} \Rightarrow |H_{z}(x, z)\cdot z|^{\kappa}\le c_1|z|^{2\kappa}\mathcal{H}(x, z), $$ where $m_0:=\min\{a_{0} , b_{0}\}$; \item[(H6')] there exist $c_0>0$ and $q\in (2, 2^*)$ such that $H(x, u, 0)\ge c_0|u|^q$ for all $(x, u)\in \mathbb{R}^{N}\times \R$; \item[(H6'')] there exist $c_0>0$ and $q\in (2, 2^*)$ such that $H(x, 0, v)\ge c_0|v|^q$ for all $(x, v)\in \mathbb{R}^{N}\times \R$. \end{itemize} \begin{remark} \label{rmk1.1}\rm It is easy to check that (H6') and (H6'') imply (H4) and (H4') with \[ T_0=\big[\frac{N^2+2}{2c_0(1-2^{-N})^2}\big]^{1/(q-2)}, \] respectively, but (H4), (H4') can not yield (H6'), (H6''). We give the following nonlinear example to illustrate it. Let $$ H(x, u, v)=(|u|^2+|v|^2)\ln(1+|u|+|v|). $$ Clearly, $H$ satisfies both (H4) and (H4') with \[ \ln (1+T_0)=\frac{N^2+2}{2\big(1-2^{-N}\big)^2}, \] but neither (H6') nor (H6''). \end{remark} \begin{example} \label{examp1.2}\rm Let $q\in (2, 2^*)$. Then it is easy to see that following two functions satisfy (H1)--(H3) and (H6'): \[ H(x, u, v)=a_1|u|^q+a_2|v|^q, \quad H(x, u, v)=\zeta(x)\big(|u|^2+|v|^2\big)^{q/2}, \] where $a_1, a_2>0$ and $\zeta\in C(\mathbb{R}^{N})$ with $0<\inf_{\mathbb{R}^N}\zeta\le \sup_{\mathbb{R}^N}\zeta<+\infty$. \end{example} Since $(q-2)N-2q<0$, we can let $h_0\ge 1$ be such that \begin{equation} \label{a1-1} \begin{aligned} &\frac{(q-2)\omega_N}{2Nq(qc_0)^{2/(q-2)}}\Big\{\frac{N^{2}+2(N+2)} {(N+2)(1-2^{-N})^2}\Big\}^{q/(q-2)}h_0^{[(q-2)N-2q]/(q-2)} \\ &= \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}, \end{aligned} \end{equation} where $\gamma_0$ and $\gamma_{2^*}$ are embedding constants, see \eqref{b0-1} and \eqref{b0-2}. If $a$ and $b$ satisfy (A0), we can choose $\lambda_0>1$ such that \begin{equation}\label{a1-2} \sup_{\lambda^{1/2}|x|\le 2h_0}|a(x)| \le h_0^{-2}, \quad \forall \lambda\ge \lambda_0, \end{equation} if $a$ and $b$ satisfy (B0), we can choose $\lambda_0>1$ such that \begin{equation}\label{a1-3} \sup_{\lambda^{1/2}|x|\le 2h_0}|b(x)|\le h_0^{-2}, \quad \forall \lambda\ge \lambda_0. \end{equation} Letting $\varepsilon^{-2}=\lambda$, \eqref{PB1} is rewritten as \begin{equation}\label{PB2} \begin{gathered} -\Delta u+\lambda a(x) u=\lambda H_{u}(x, u, v)+\lambda \mu(x) v, \quad x\in \mathbb{R}^N,\\ -\Delta v+\lambda b(x) v=\lambda H_{v}(x, u, v)+\lambda \mu(x) u, \quad x\in \mathbb{R}^N,\\ u,v\in H^1(\mathbb{R}^N). \end{gathered} \end{equation} Let \begin{equation} \label{PB3} \begin{aligned} \Phi_{\lambda}(z) &= \frac{1}{2}\int_{\mathbb{R}^N}(|\nabla u|^2+|\nabla v|^2 +\lambda a(x)|u|^2+\lambda b(x)|v|^2)\,\mathrm{d}x \\ & \quad -\lambda \int_{\mathbb{R}^N}H(x, z)\,\mathrm{d}x -\lambda \int_{\mathbb{R}^N}\mu(x)u v\,\mathrm{d}x, \quad z=(u, v). \end{aligned} \end{equation} Obviously, the solutions of \eqref{PB1} are the critical points of $\Phi_{\varepsilon^{-1/2}}(z)$; the solutions of \eqref{PB2} are the critical points of $\Phi_{\lambda}(z)$. We are now in a position to state the main results of this paper. \begin{theorem} \label{thm1.3} Assume that $a$, $b$, $\mu$ and $H$ satisfy {\rm (A0), (A1), (H1)--(H5)}. Then for $0<\varepsilon\le \lambda_0^{-1/2}$, \eqref{PB1} has a solution $z_{\varepsilon}=(u_{\varepsilon}, v_{\varepsilon})$ such that \begin{gather*} 0<\Phi_{\varepsilon^{-1/2}}(z_{\varepsilon}) \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\varepsilon^{N-2}, \\ \int_{\mathbb{R}^N}\mathcal{H}(x, z_{\varepsilon})\,\mathrm{d}x \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\varepsilon^{N}. \end{gather*} \end{theorem} \begin{theorem} \label{thm1.4} Assume that $a$, $b$, $\mu$ and $H$ satisfy {\rm (A1), (B0), (H1)--(H3), (H4'), (H5)}. Then for $0<\varepsilon\le \lambda_0^{-1/2}$, \eqref{PB1} has a solution $z_{\varepsilon}=(u_{\varepsilon}, v_{\varepsilon})$ such that \begin{gather*} 0<\Phi_{\varepsilon^{-1/2}}(z_{\varepsilon}) \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\varepsilon^{N-2}, \\ \int_{\mathbb{R}^N}\mathcal{H}(x, z_{\varepsilon})\,\mathrm{d}x\le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\varepsilon^{N}. \end{gather*} \end{theorem} \begin{theorem} \label{thm1.5} Assume that $a$, $b$, $\mu$ and $H$ satisfy {\rm (A0), (A1), (H1)--(H5)}. Then for $\lambda\geq \lambda_0$, \eqref{PB2} has a solution $z_{\lambda}=(u_{\lambda}, v_{\lambda})$ such that \begin{gather*} 0<\Phi_{\lambda}(z_{\lambda}) \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\lambda^{1-N/2}, \\ \int_{\mathbb{R}^N}\mathcal{H}(x, z_{\lambda})\,\mathrm{d}x \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\lambda^{-N/2}. \end{gather*} \end{theorem} \begin{theorem} \label{thm1.6} Assume that $a$, $b$, $\mu$ and $H$ satisfy {\rm (A1), (B0), (H1)--(H3), (H4'), (H5)}. Then for $\lambda\geq \lambda_0$, \eqref{PB2} has a solution $z_{\lambda}=(u_{\lambda}, v_{\lambda})$ such that \begin{gather*} 0<\Phi_{\lambda}(z_{\lambda}) \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\lambda^{1-N/2}, \\ \int_{\mathbb{R}^N}\mathcal{H}(x, z_{\lambda})\,\mathrm{d}x \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\lambda^{-N/2}. \end{gather*} \end{theorem} The rest of the article is organized as follows. In Section 2, we provide some preliminaries and lemmas. In Section 3, we give the proofs of Theorems \ref{thm1.3}--\ref{thm1.6}. \section{Preliminaries} Let \begin{gather*} E=\big\{(u ,v)\in H^{1}(\mathbb{R}^N)\times H^{1}(\mathbb{R}^N): \int_{\mathbb{R}^N} [a(x)|u|^2+b(x)|v|^2]\,\mathrm{d}x< +\infty \big\}, \\ \|z\|_{\lambda\dag}=\left\{\int_{\mathbb{R}^N} [|\nabla{u}|^2 +\lambda a(x)|u|^2+|\nabla{v}|^2+\lambda b(x)|v|^2]\,\mathrm{d}x\right\}^{1/2}, \quad \forall z=(u ,v)\in E. \end{gather*} Analogous to the proof of \cite[Lemma 1]{Si}, by using (A0) or (B0) and the Sobolev inequality, one can show that there exists a constant $\gamma_0>0$ independent of $\lambda$ such that \begin{equation}\label{b0-1} \|z\|_{H^{1}(\mathbb{R}^N)}\le \gamma_0\|z\|_{\lambda\dag}, \quad \forall z\in E, \; \lambda \ge 1. \end{equation} This shows that $(E, \|\cdot\|_{\lambda\dag})$ is a Banach space for $\lambda\ge 1$. Furthermore, by the Sobolev embedding theorem, we have \begin{equation}\label{b0-2} \|z\|_{s}\le \gamma_s\|z\|_{H^{1}(\mathbb{R}^N)} \le \gamma_s\gamma_0\|z\|_{\lambda\dag}, \quad \forall z\in E, \; \lambda\ge 1, \; 2\le s\le 2^*, \end{equation} here and in the sequel, we denote by $\|\cdot\|_s$ the usual norm in space $L^s(\mathbb{R}^N)$. In view of the definition of the norm $\|\cdot\|_{\lambda\dag}$, we can re-write $\Phi_{\lambda}$ in the form \begin{equation}\label{PB4} \Phi_{\lambda}(z)=\frac{1}{2}\|z\|_{\lambda\dag}^2 -\lambda\int_{\mathbb{R}^N}H(x, z)\,\mathrm{d}x -\lambda\int_{\mathbb{R}^N}\mu(x)uv\,\mathrm{d}x, \quad \forall z\in E. \end{equation} It is easy to see that $\Phi_{\lambda} \in C^1(E,\mathbb{R})$ and \begin{equation} \label{PB5} \begin{aligned} \langle\Phi_{\lambda}'(z), \tilde{z}\rangle &= \int_{\mathbb{R}^N}[\nabla u\cdot\nabla \tilde{u} +\nabla v\cdot\nabla \tilde{v}+\lambda a(x)u \tilde{u} +\lambda b(x)v \tilde{v}]\,\mathrm{d}x \\ &\quad -\lambda\int_{\mathbb{R}^N}[H_u(x ,z) \tilde{u} +H_v(x ,z) \tilde{v}]\,\mathrm{d}x \\ &\quad -\lambda\int_{\mathbb{R}^N}\mu(x)(u \tilde{v}+v \tilde{u})]\,\mathrm{d}x, \quad \forall z=(u,v),\; \tilde{z}=(\tilde{u},\tilde{v})\in E. \end{aligned} \end{equation} As in \cite{LT}, we let \begin{equation}\label{th} \vartheta(x): =\begin{cases} \frac{1}{h_0}, & |x|\le h_0,\\[4pt] \frac{h_0^{N-1}}{1-2^{-N}}[|x|^{-N}-(2h_0)^{-N}], & h_0<|x|\le 2h_0,\\[4pt] 0, & |x|>2h_0. \end{cases} \end{equation} Then $\vartheta\in H^{1}(\mathbb{R}^N)$, moreover, \begin{gather}\label{th1} \|\nabla\vartheta\|_2^2 = \int_{\mathbb{R}^N}|\nabla \vartheta(x)|^2\,\mathrm{d}x \le \frac{N^2\omega_N}{(N+2)(1-2^{-N})^2}h_0^{N-4},\\ \label{th2} \|\vartheta\|_2^2 = \int_{\mathbb{R}^N}|\vartheta(x)|^2\,\mathrm{d}x \le \frac{2\omega_N}{(1-2^{-N})^2N}h_0^{N-2}. \end{gather} Let $e_{\lambda}(x)=\vartheta(\lambda^{1/2}x)$. Then we can prove the following lemma which is used for our proofs. \begin{lemma} \label{lem2.1} Let $H(x, z)\ge 0$, for all $(x, z)\in \mathbb{R}^N\times \mathbb{R}^2$. Suppose that {\rm (A0), (A1), (H1)--(H4)} are satisfied. Then \begin{equation}\label{b1-1} \sup\{\Phi_{\lambda}(se_{\lambda}, 0) : s\ge 0\} \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\lambda^{1-N/2}, \quad \forall \lambda\ge \lambda_0. \end{equation} \end{lemma} \begin{proof} From (H4), \eqref{a1-1}, \eqref{a1-2} ,\eqref{PB3}, \eqref{th}, \eqref{th1} and \eqref{th2}, we obtain \begin{equation} \label{b1-2} \begin{aligned} &\Phi_{\lambda}(se_{\lambda} ,0)\\ &= \frac{s^2}{2}\int_{\mathbb{R}^N}(|\nabla e_{\lambda}|^2 +\lambda a(x)|e_{\lambda}|^2)\,\mathrm{d}x -\lambda\int_{\mathbb{R}^N}H(x, se_{\lambda}, 0)\,\mathrm{d}x \\ &= \lambda^{1-N/2}\Big[\frac{s^2}{2}\int_{\mathbb{R}^N}(|\nabla \vartheta|^2 +a(\lambda^{-1/2}x)|\vartheta|^2)\,\mathrm{d}x -\int_{\mathbb{R}^N}H(\lambda^{-1/2}x, s\vartheta, 0)\,\mathrm{d}x\Big] \\ &\leq \lambda^{1-N/2}\Big[\frac{s^2}{2}\Big(\|\nabla \vartheta\|_2^2 +\|\vartheta\|_2^2\sup_{|x|\le 2h_0}|a(\lambda^{-1/2}x)|\Big)\\ &\quad -\int_{|x|\le h_0}H(\lambda^{-1/2}x, s/h_0, 0)\,\mathrm{d}x\Big] \\ &\leq \lambda^{1-N/2}\Big[\frac{s^2}{2}(\|\nabla \vartheta\|_2^2+h_0^{-2}\|\vartheta\|_2^2) -\int_{|x|\le h_0}H(\lambda^{-1/2}x, s/h_0, 0)\,\mathrm{d}x\Big], \\ &\quad \forall s\ge 0, \; \lambda\ge \lambda_0, \end{aligned} \end{equation} \begin{equation} \label{b1-3} \begin{aligned} &\frac{s^2}{2}(\|\nabla \vartheta\|_2^2+h_0^{-2}\|\vartheta\|_2^2) -\int_{|x|\le h_0}H(\lambda^{-1/2}x, s/h_0, 0)\,\mathrm{d}x \\ &\leq \frac{s^2}{2}[\|\nabla \vartheta\|_2^2+h_0^{-2}\|\vartheta\|_2^2 -\frac{(N^2+2)\omega_N}{N(1-2^{-N})^2}h_0^{N-4}]\le 0, \quad \forall s\ge h_0T_0, \; \lambda\ge \lambda_0 \end{aligned} \end{equation} and \begin{equation}\label{b1-4} \begin{aligned} &\frac{s^2}{2}(\|\nabla \vartheta\|_2^2+h_0^{-2}\|\vartheta\|_2^2) -\int_{|x|\le h_0}H(\lambda^{-1/2}x, s/h_0, 0)\,\mathrm{d}x \\ &\leq \frac{s^2}{2}(\|\nabla \vartheta\|_2^2 +h_0^{-2}\|\vartheta\|_2^2)-\frac{c_0\omega_N}{N}s^qh_0^{N-q} \\ &\leq \frac{(q-2)(\|\nabla \vartheta\|_2^2+h_0^{-2}\|\vartheta\|_2^2)^{q/(q-2)}} {2q(\frac{qc_0\omega_N}{N}h_0^{N-q})^{2/(q-2)}} \\ &\leq \frac{(q-2)\omega_N}{2Nq(qc_0)^{2/(q-2)}}\Big\{\frac{N^2+2(N+2)} {(N+2)(1-2^{-N})^2}\Big\}^{q/(q-2)}h_0^{[(q-2)N-2q]/(q-2)} \\ &= \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}, \quad \forall 0\le s\le h_0T_0, \quad \lambda\ge \lambda_0. \end{aligned} \end{equation} The conclusion of Lemma \ref{lem2.1} follows from \eqref{b1-2}, \eqref{b1-3} and \eqref{b1-4}. \end{proof} We can prove the following lemma in the same way as Lemma \ref{lem2.1}. \begin{lemma} \label{lem2.2} Let $H(x, z)\ge 0$ for all $(x, z)\in \mathbb{R}^N\times \mathbb{R}^2$. Suppose that {\rm (A1), (B0), (H1)--(H3), (H4')} are satisfied. Then \begin{equation}\label{b2-2} \sup\{\Phi_{\lambda}(0, se_{\lambda}) : s\ge 0\} \le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\lambda^{1-N/2}, \quad \forall \lambda\ge \lambda_0. \end{equation} \end{lemma} Applying the mountain-pass lemma without the (PS) condition, by standard arguments, we can prove the following two lemmas. \begin{lemma} \label{lem2.3} Let $H(x, z)\ge 0$ for all $(x, z)\in \mathbb{R}^N\times \mathbb{R}^2$. Suppose that {\rm (A0), (A1), (H1)--(H4)} are satisfied. Then there exist a constant $d_{\lambda}\in (0, \sup_{s\ge 0} \Phi_{\lambda}(se_{\lambda}, 0)]$ and a sequence $\{z_n\}\subset E$ satisfying \begin{equation}\label{Ce1} \Phi_{\lambda}(z_n)\to d_{\lambda}, \quad \|\Phi_{\lambda}'(z_n)\|_{E^*}(1+\|z_n\|_{\lambda\dag})\to 0. \end{equation} \end{lemma} \begin{lemma} \label{lem2.4} Let $H(x, z)\ge 0$ for all $(x, z)\in \mathbb{R}^N\times \mathbb{R}^2$. Suppose {\rm (A1), (B0), (H1)--(H3), (H4')} are satisfied. Then there exist a constant $d_{\lambda}\in (0, \sup_{s\ge 0} \Phi_{\lambda}(0, se_{\lambda})]$ and a sequence $\{z_n\}\subset E$ satisfying \begin{equation}\label{Ce2} \Phi_{\lambda}(z_n)\to d_{\lambda}, \quad \|\Phi_{\lambda}'(z_n)\|_{E^*}(1+\|z_n\|_{\lambda\dag})\to 0. \end{equation} \end{lemma} \begin{lemma} \label{lem2.5} Suppose that {\rm(A0), (A1), (H1)--(H5)} are satisfied. Then any sequence $\{z_n\}\subset E$ satisfying \eqref{Ce1} is bounded in $E$. \end{lemma} \begin{proof} We argue by contradiction for proving boundedness of $\{z_n\}$. Suppose that $\|z_n\|_{\lambda\dag} \to \infty$. Let $\tilde{z}_n=z_n/\|z_n\|_{\lambda\dag}:=(\tilde{u}_n, \tilde{v}_n)$. Then $\|\tilde{z}_n\|_{\lambda\dag}=1$. In view of (A1), we obtain \begin{equation} \label{b3-0} \begin{aligned} 2\lambda\int_{\mathbb{R}^N}\mu(x)\tilde{u}_n \tilde{v}_n\,\mathrm{d}x &\leq 2\theta\lambda\int_{\mathbb{R}^N}\sqrt{a(x)b(x)}|\tilde{u}_n \tilde{v}_n|\,\mathrm{d}x \\ &\leq \theta\lambda\int_{\mathbb{R}^N}[a(x)\tilde{u}_n^2+b(x)\tilde{v}_n^2]\,\mathrm{d}x \le \theta. \end{aligned} \end{equation} If $$ \delta:=\limsup_{n\to\infty}\sup_{y\in \mathbb{R}^N} \int_{B(y,1)}|\tilde{z}_n|^2\,\mathrm{d}x=0, $$ then by Lions' concentration compactness principle \cite{Lio} or \cite[Lemma 1.21]{Wi}, $\tilde{z}_n\to (0, 0)$ in $L^{s}(\mathbb{R}^N)$ for $20$. Going to a subsequence if necessary, we assume the existence of $k_n\in \Z^N$ such that $\int_{B_{1+\sqrt{N}}(k_n)}|\tilde{z}_n|^2dx > \frac{\delta}{2}$. Let $w_n(x)=\tilde{z}_n(x+k_n)$; then \begin{equation}\label{b3-5} \int_{B_{1+\sqrt{N}}(0)}|w_n|^2dx> \frac{\delta}{2}. \end{equation} Now we define $\hat{z}_n(x)=z_n(x+k_n)$, then $\hat{z}_n/\|z_n\|_{\lambda\dag}=w_n$ and $\|w_n\|_{H^{1}(\mathbb{R}^N)}^2 =\|\tilde{z}_n\|_{H^{1}(\mathbb{R}^N)}^2$. Passing to a subsequence, we have $w_n\rightharpoonup w$ in $H^{1}(\mathbb{R}^N)$, $w_n\to w$ in $L^{s}_{\mathrm{loc}}(\mathbb{R}^N)$, $2\le s<2^*$ and $w_n\to w$ a.e. on $\mathbb{R}^N$. Obviously, \eqref{b3-5} implies that $w\ne (0, 0)$. For a.e. $x\in \{z\in \mathbb{R}^N : w(z)\ne (0, 0)\}$, we have $\lim_{n\to\infty}|\hat{z}_n(x)|=\infty$. Hence, it follows from (H3), \eqref{PB4}, \eqref{Ce1}, \eqref{b3-0} and Fatou's lemma that \begin{align*} 0 &= \lim_{n\to\infty}\frac{d_{\lambda}+o(1)}{\|z_n\|_{\lambda\dag}^2} = \lim_{n\to\infty}\frac{\Phi_{\lambda}(z_n)}{\|z_n\|_{\lambda\dag}^2}\\ &= \lim_{n\to\infty}\Big[\frac{1}{2}\|\tilde{z}_n\|_{\lambda\dag}^2 -\lambda\int_{\mathbb{R}^N}\frac{H(x+k_n, \hat{z}_n)}{|\hat{z}_n|^2}|w_n|^2 \,\mathrm{d}x -\lambda\int_{\mathbb{R}^N}\mu(x)\tilde{u}_n \tilde{v}_n\,\mathrm{d}x\Big]\\ &\leq \frac{1+\theta}{2}-\lambda\int_{\mathbb{R}^N}\liminf_{n\to\infty} \frac{H(x+k_n, \hat{z}_n)}{|\hat{z}_n|^2}|w_n|^2\,\mathrm{d}x = -\infty. \end{align*} This contradiction shows that $\{\|z_n\|_{\lambda\dag}\}$ is bounded. \end{proof} We can prove the following lemma in the same way as Lemma \ref{lem2.5}. \begin{lemma} \label{lem2.6} Suppose that {\rm (A1), (B0), (H1)--(H3), (H4'), (H5)} are satisfied. Then any sequence $\{z_n\}\subset E$ satisfying \eqref{Ce2} is bounded in $E$. \end{lemma} \section{Proofs of main results} In this section, we give the proofs of Theorems \ref{thm1.3}--\ref{thm1.6}. \begin{proof}[Proof of Theorem \ref{thm1.5}] Applying Lemmas \ref{lem2.1}, \ref{lem2.3} and \ref{lem2.5}, we deduce that there exists a bounded sequence $\{z_n\}\subset E$ satisfying \eqref{Ce1} with \begin{equation}\label{c1-1} d_{\lambda}\le \frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\lambda^{1-N/2}, \quad \forall \lambda\ge \lambda_0. \end{equation} Going to a subsequence, if necessary, we can assume that $z_n\rightharpoonup z_{\lambda}$ in $(E, \|\cdot\|_{\lambda\dag})$ and $\Phi_{\lambda}'(z_n)\to 0$. Next, we prove that $z_{\lambda}\ne (0, 0)$. Arguing by contradiction, suppose that $z_{\lambda} = (0, 0)$, i.e. $z_n\rightharpoonup (0, 0)$ in $E$, and so $z_n\to (0, 0)$ in $L^{s}_{\mathrm{loc}}(\mathbb{R}^N)$, $2\le s<2^*$ and $z_n\to (0, 0)$ a.e. on $\mathbb{R}^N$. Since $\mathcal{D}$ is a set of finite measure, there holds \begin{equation}\label{c1-2} \|z_n\|_2^2=\int_{\mathbb{R}^N\setminus \mathcal{D}}|z_n|^2\,\mathrm{d}x +\int_{\mathcal{D}}|z_n|^2\,\mathrm{d}x \le \frac{1}{\lambda m_0}\|z_n\|_{\lambda\dag}^2+o(1). \end{equation} For $s\in (2, 2^*)$, it follows from \eqref{b0-2}, \eqref{c1-2} and the H\"older inequality that \begin{equation}\label{c1-3} \begin{aligned} \|z_n\|_s^s &\leq \|z_n\|_2^{2(2^*-s)/(2^*-2)}\|z_n\|_{2^*}^{2^*(s-2)/(2^*-2)} \\ &\leq (\gamma_{2^*}\gamma_0)^{2^*(s-2)/(2^*-2)} (\lambda m_0)^{-(2^*-s)/(2^*-2)}\|z_n\|_{\lambda\dag}^s+o(1). \end{aligned} \end{equation} According to \eqref{c1-2}, one can obtain that \begin{equation}\label{c1-4} \begin{aligned} \lambda\int_{\Omega_n} H_z(x, z_n)\cdot z_n\,\mathrm{d}x &= \lambda\int_{\Omega_n}\frac{H_z(x, z_n)\cdot z_n}{|z_n|^2}|z_n|^2\,\mathrm{d}x \\ &\leq \frac{(1-\theta)\lambda m_0}{3}\|z_n\|_2^2 \leq \frac{1-\theta}{3}\|z_n\|_{\lambda\dag}^2+o(1). \end{aligned} \end{equation} By \eqref{PB4}, \eqref{PB5} and \eqref{Ce1}, we have \begin{equation}\label{c1-5} \Phi_{\lambda}(z_n)-\frac{1}{2}\langle\Phi_{\lambda}'(z_n), z_n\rangle=\lambda\int_{\mathbb{R}^N}\mathcal{H}(x, z_n)\,\mathrm{d}x =d_{\lambda}+o(1). \end{equation} Using (H5), \eqref{c1-1}, \eqref{c1-3} with $s=2\kappa/(\kappa-1)$ and \eqref{c1-5}, we obtain \begin{equation} \label{c1-6} \begin{aligned} & \lambda\int_{\mathbb{R}^N\setminus \Omega_n}H_z(x, z_n)\cdot z_n\,\mathrm{d}x \\ &\leq \lambda\Big(\int_{\mathbb{R}^N\setminus \Omega_n} \Big(\frac{|H_z(x, z_n)\cdot z_n|} {|z_n|^2}\Big)^{\kappa}\,\mathrm{d}x \Big)^{1/\kappa}\|z_n\|_s^2 \\ &\leq (\gamma_{2^*}\gamma_0)^{2\cdot 2^*(s-2)/s(2^*-2)}\lambda \Big(c_1\int_{\mathbb{R}^N\setminus \Omega_n} \mathcal{H}(x, z_n)\,\mathrm{d}x\Big)^{1/\kappa}\\\ &\quad\times (\lambda m_0)^{-2(2^*-s)/s(2^*-2)} \|z_n\|_{\lambda\dag}^2+o(1) \\ &\leq c_1^{1/\kappa}(\gamma_{2^*}\gamma_0)^{N/\kappa} \lambda^{1-1/\kappa}d_{\lambda}^{1/\kappa} (\lambda m_0)^{-2(2^*-s)/s(2^*-2)}\|z_n\|_{\lambda\dag}^2+o(1) \\ &= \frac{c_1^{1/\kappa}(\gamma_{2^*}\gamma_0)^{N/\kappa}}{m_0^{(2\kappa-N)/2\kappa}} \big[\lambda^{(N-2)/2}d_{\lambda}\big]^{1/\kappa}\|z_n\|_{\lambda\dag}^2+o(1) \\ &\leq \frac{c_1^{1/\kappa}(\gamma_{2^*}\gamma_0)^{N/\kappa}} {m_0^{(2\kappa-N)/2\kappa}} \big[\frac{(1-\theta)^{\kappa} m_0^{(2\kappa-N)/2}} {3^{\kappa}c_1(\gamma_{2^*}\gamma_0)^{N}}\big]^{1/\kappa} \|z_n\|_{\lambda\dag}^2+o(1) \\ & = \frac{1-\theta}{3}\|z_n\|_{\lambda\dag}^2+o(1), \end{aligned} \end{equation} which, together with \eqref{PB5}, \eqref{Ce1} and \eqref{c1-4}, yields \begin{equation} \label{c1-7} \begin{aligned} o(1) &= \langle\Phi_{\lambda}'(z_n), z_n\rangle \\ &= \|z_n\|_{\lambda\dag}^2-\lambda\int_{\mathbb{R}^N}H_z(x, z_n)\cdot z_n\,\mathrm{d}x -2\lambda\int_{\mathbb{R}^N}\mu(x)u_n v_n\,\mathrm{d}x \\ & \ge (1-\theta)\|z_n\|_{\lambda\dag}^2-\lambda\int_{\Omega_n}H_z(x, z_n)\cdot z_n\,\mathrm{d}x -\lambda\int_{\mathbb{R}^N\setminus \Omega_n}H_z(x, z_n)\cdot z_n\,\mathrm{d}x \\ & \ge \frac{1-\theta}{3}\|z_n\|_{\lambda\dag}^2+o(1). \end{aligned} \end{equation} Consequently, it follows from \eqref{PB4} and \eqref{Ce1} that $$ 0