\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2016 (2016), No. 102, pp. 1--22.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2016 Texas State University.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2016/102\hfil Nontrivial solutions] {Nontrivial solutions for Kirchhoff equations with periodic potentials} \author[X. Ma, X. He \hfil EJDE-2016/102\hfilneg] {Xiaoyan Ma, Xiaoming He} \address{Xiaoyan Ma \newline College of Science, Minzu University of China, Beijing 100081, China} \email{SX140963@muc.edu.cn} \address{Xiaoming He \newline College of Science, Minzu University of China, Beijing 100081, China} \email{xmhe923@muc.edu.cn} \thanks{Submitted March 28, 2016. Published April 20, 2016.} \subjclass[2010]{47G20, 35J50, 35B65} \keywords{Kirchhoff-type problems; ground states; Nehari manifold; \hfill\break\indent critical Sobolev exponent} \begin{abstract} In this article we consider the Kirchhoff equations $$ -\Big(a+b\int_{\mathbb{R}^3}|\nabla u|^2\Big)\Delta u+V(x)u=f(x,u),\quad x\in\mathbb{R}^3, $$ where $a,b>0$ are constants, the nonlinearity $f$ is superlinear at infinity with subcritical or critical growth and $V$ is positive, continuous and periodic in $x$. Some existence results for ground state solutions are obtained by using variational methods. Moreover, when $V\equiv 1$ we obtain ground state solutions for the above problem with a wide class of superlinear nonlinearities by using a new approach. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{remark}[theorem]{Remark} \allowdisplaybreaks \section{Introduction and main results} In this article we study the Kirchhoff-type equation \begin{equation}\label{e1.1} -\Big(a+b\int_{\mathbb{R}^3}|\nabla u|^2\Big)\Delta u+V(x)u=f(x,u),\quad x\in\mathbb{R}^3, \end{equation} where $a,b>0$ are constants, and $f\in C(\mathbb{R}^3\times\mathbb{R},\mathbb{R})$ satisfies some conditions which will be stated later. Equations of the form \eqref{e1.1} have been extensively studied because of their interesting physical context. \eqref{e1.1} is often referred to be a nonlocal problem in view of that the appearance of the term $\int_{\mathbb{R}^3}|Du|^2$ implies that \eqref{e1.1} is not a point wise identity. This causes some mathematical difficulties which make the study of \eqref{e1.1} particularly interesting. When we set $V=0$ and replace $\mathbb{R}^N$ by a bounded domain $\Omega\subset\mathbb{R}^N$ in \eqref{e1.1}, we get the Kirchhoff-type Dirichlet problem \begin{equation} \label{e1.2} \begin{gathered} -\Big(a+b\int_{\Omega}|\nabla u|^2\Big)\Delta u=f(x,u),\quad x\in\Omega, \\ u=0,\quad x\in\partial\Omega, \end{gathered} \end{equation} which is related to the stationary analogue of the Kirchhoff equation \begin{equation} \label{e1.3} u_{tt}-\Big(a+b\int_{\Omega}|\nabla u|^2\Big)\Delta u=f(x,u), \end{equation} which was proposed by Kirchhoff \cite{K} as an extension of classical D'Alembert's wave equation for free vibrations of elastic strings. Lions \cite{Lions} introduced an abstract functional analysis framework to the above equation. After that, \eqref{e1.3} has been receiving much attention, see \cite{AF,AP,Azzollini,Azzollini1,CDS,AS} and the references therein. We remark that the stationary problem associated with \eqref{e1.3}, i.e., \eqref{e1.2} has been investigated by many researchers by using variational methods, see for example, \cite{AC,ACM,CKW,HZ3,MR,MZ,PZ,ZP} and the references therein. Especially, Ma and Rivera \cite{MR} showed the existence and non-existence of positive solutions for a class of Kirchhoff type equation by variational methods. The existence of multiple positive solutions for \eqref{e1.2} was also proved in \cite{ACM,CKW,HZ3,PZ}. Mao and Zhang \cite{MZ} investigated the existence of sign-changing solutions for \eqref{e1.2} by using minimax methods and invariant sets of descent flow. We also recall that in recent years, there have been new results on existence, nonexistence and multiplicity of solutions for the following parameter-perturbed Kirchhoff equation \begin{equation}\label{e1.4} \begin{gathered} -\Big(\varepsilon^2a+\varepsilon b\int_{\mathbb{R}^N}|\nabla u|^2\Big)\Delta u+V(x)u=f(x,u),\quad x\in\mathbb{R}^3, \\ u\in H^1(\mathbb{R}^3),\quad u(x)>0\quad x\in\mathbb{R}^3, \end{gathered} \end{equation} where $\varepsilon>0$ is a parameter. Jin and Wu \cite{JW}, proved that \eqref{e1.4} has infinitely many radial solutions by using a fountain theorem \cite{W} when $\varepsilon=1, V(x)\equiv1, f(x,u)$ is subcritical, superlinear at the origin and 4-superlinear at infinity. Wu \cite{Wu} obtained the existence of a sequence of high energy solutions \eqref{e1.4} with $\varepsilon=1$, by applying a Symmetric Mountain Pass Theorem \cite{R2}, the potential $V(x)\in C(\mathbb{R}^3,\mathbb{R})$ is assumed to satisfy \begin{itemize} \item $V\in C(\mathbb{R}^3,\mathbb{R})$ satisfies $\inf_{x\in\mathbb{R}^3}V(x)\geq a_1>0$ and for each $M>0, \text{meas}\{x\in\mathbb{R}^3: V(x)\leq M\}<+\infty$, where $a_1$ is a constant and meas denotes the Lebesgue measure in $\mathbb{R}^3$. \end{itemize} For $\varepsilon$ small, He and Zou \cite{HZ1,HZ2}, proved the existence, multiplcity and concentration of positive solutions of \eqref{e1.4} by using Ljusternik-Schnirelmann category theory, Nehari manifold. For other existence and concentration results, we refer to He, Li and Peng \cite{HLP}, Figueiredo, Ikoma and Santos \cite{FIS}, Li and Ye \cite{LY2}, Wang, Tian, Xu and Zhang \cite{WTXZ}, Sun and Ma \cite{SM} and the references therein. In \cite{HZ1,HZ2,WTXZ}, the potential $V$ is required to satisfy the following Rabinowitz-type condition \cite{R1} \begin{itemize} \item $0\inf_{\Lambda}:=V_0. $$ \end{itemize} Motivated by the above works, we consider problem \eqref{e1.1} with periodic potential $V$ and more general assumptions on $f$ which may subcritical or critical growth at infinity but not need to be $C^1$. We first consider the subcritical case. We use the following assumptions on $V$ and $f$: \begin{itemize} \item[(A1)] $V$ is continuous, 1-periodic in $x_i$, for $i=1,2,3$. and $V_0=\inf_{x\in \mathbb{R}^3}V(x)>0$. \item[(A2)] $f\in C(\mathbb{R}^3\times \mathbb{R},\mathbb{R})$, $f$ is 1-periodic in $x_i$, for $i=1,2,3$ and $|f(x,u)|\leq C(1+|u|^{p-1})$ for some $C>0$ and $p\in(4,6)$; \item[(A3)] $f(x,u)=o(u)$ uniformly in $x$ as $u\to 0$; \item[(A4)] $F(x,u)/u^4$ uniformly in $x$ as $u\to \infty$; \item[(A5)] $u\to f(x,u)/u^3$ is positive for $u\neq 0$, strictly decreasing on $(-\infty,0)$ and strictly increasing on $(0,\infty)$. \end{itemize} Our first main result reads as follows. \begin{theorem} \label{thm11} Suppose {\rm (A1)--(A5)} are satisfied. Then \begin{itemize} \item[(i)] Problem \eqref{e1.1} has a ground state solution; \item[(ii)] $\mathcal {K}$ is compact (up to translation ) in $H^1(R^3)$, where $\mathcal {K}$ denotes the set of all ground state solutions of \eqref{e1.1}. \end{itemize} \end{theorem} We next study the existence of ground state solutions of problem \eqref{e1.2} with the critical growth case. To be precise, we consider the problem \begin{equation}\label{e1.5} -\Big(a+b\int_{\mathbb{R}^3}|\nabla u|^2\Big)\Delta u+V(x)u =K(x)|u|^4u+\lambda g(x,u)\quad \text{in } \mathbb{R}^3, \end{equation} where $\lambda>0$ is a real number. Let $G(x,u)=\int_0^u g(x,s)ds$, assume that $V$ satisfies (A1) and $K$ and $g$ satisfy the following assumptions: \begin{itemize} \item[(A6)] $K$ is continuous, 1-periodic in $x_i$, for $i=1,2,3$, $K(x)>0$ for all $x\in \mathbb{R}^3$ and $K(x)-K(x_0)=O(|x-x_0|^\alpha)$ as $x\to x_0$, where $\alpha>0$, $K(x_0)=\max_{\mathbb{R}^3}K(x)$; \item[(A7)] $g\in C(\mathbb{R}^3\times \mathbb{R},\mathbb{R})$, $g$ is 1-periodic in $x$ and $|g(x,u)|\leq C(1+|u|^{p-1})$ for some $C>0$ and $p\in (2,6)$; \item[(A8)] $g(x,u)= o(u)$ uniformly in $x$ as $u\to 0$; \item[(A9)] $u\to g(x,u)/u^3$ is positive for $u\neq 0$, nonincreasing on $(-\infty,0)$ and nodecreaing on $(0,\infty)$. \end{itemize} The main result for \eqref{e1.5} is states as follows. \begin{theorem} \label{thm12} Suppose assumptions {\rm (A1), (A6)--(A9)} hold. (i) If $\alpha\geq1$ and $g$ satisfies: \begin{itemize} \item[(A10)] There are $c_0>0$ and $q \geq 4$ such that $G(x,u)\geq c_0|u|^q$ for all $(x,u)$, \end{itemize} then problem \eqref{e1.5} has a ground state solution for any $\lambda>0$ whenever $q\in (4,6)$; problem \eqref{e1.5} admits a ground state solution provided that $\lambda$ is sufficiently large whenever $q=4$. (ii) If $\alpha\in(0,1)$ and $g$ satisfies: \begin{itemize} \item[(A11)] there exists an open set $\Omega \subset \mathbb{R}^3 $ with $x_0\in \Omega$ such that $$ \lim_{|s|\to\infty}\frac{G(x,s)}{|s|^{2(3-\alpha)}}=+\infty, $$ \end{itemize} then \eqref{e1.5} has a ground state solution for any $\lambda>0$. (iii) Under the assumption of $(i)$ or $(ii)$, $\widetilde{\mathcal {K}}$ is compact (up to translation) in $H^1(\mathbb{R}^3)$, where $\widetilde{\mathcal {K}}$ denotes the set of all ground state solutions of \eqref{e1.5}. \end{theorem} \begin{remark} \label{ma-he11} \rm (i) Clearly, $G(x,u)=|u|^q$ for $q\geq 4$ and $G(x,u)=|u|^{6-2\alpha+\delta}$ with $\alpha\in (0,1)$ and $\delta\in (0,2\alpha)$ satisfy (A10) and (A11) respectively. (ii) The condition (A11) was first given in \cite{GL} to deal with the general critical growth semilinear elliptic equations on bounded domains. (iii) Note that if $V(x)\equiv1$, the conclusions of Theorems \ref{thm11} and \ref{thm12} remain valid. \end{remark} Our argument in this article is variational. Let $E :=H^1(\mathbb{R}^3)$, under the periodic assumption (A1), we define a new norm $$ \|u\| :=\Big(\int_{\mathbb{R}^3}(a|\nabla u|^2+V(x)u^2 dx)\Big)^{1/2} $$ in $E$ which is equivalent to the usual norm of $E$. Denote the norm of $D^{1,2}$ by $$\ |u\|_{D^{1,2}} :=\Big(\int_{\mathbb{R}^3} (|\nabla u|^2dx\Big)^{1/2}. $$ Moreover, under our assumption it is standard to see that the solutions of \eqref{e1.1} correspond to the critical points of the functional defined in $E$ by $$ I(u)=\frac{1}{2}\int_{\mathbb{R}^3} (a|\nabla u|^2+V(x)u^2)dx+\frac{b}{4}\Big(\int_{\mathbb{R}^3} |\nabla u|^2dx\Big)^2-\int_{\mathbb{R}^3} F(x,u)dx,\quad \forall u\in E. $$ Hence if $u\in E$ is a critical point of $I$, then the $u$ is a solution of \eqref{e1.1}. To prove Theorems \ref{thm11} and \ref{thm12}, we define the Nehari manifold of \eqref{e1.1} as the set $$ \mathcal {N} :=\{u\in E\backslash \{0\} : \langle I'(u),u \rangle =0\}. $$ Obviously, $\mathcal {N}$ contains all nontrivial critical points of $I$. We do not know whether $\mathcal {N}$ is of class $C^1$ under our assumptions and therefore we cannot use minimax methods directly on $\mathcal {N}$. To overcome this difficulty, we shall employ Szulkin and Weth's technique \cite{SW1,SW2} to show that $\mathcal {N}$ is still a topological manifold, naturally homeomorphic to the unit sphere of $E$, and then we can consider a new minimax characterization of the corresponding critical value for $I$. Finally, we try to obtain the existence of ground state solutions to \eqref{e1.1} with $V\equiv1$ and more general nonlinearity than that of \cite{JW} by using a new approach. More precisely, we consider the autonomous Kirchhoff-type problem \begin{equation} \label{e1.6} -\Big(a+b\int_{\mathbb{R}^3}|\nabla u|^2\Big)\Delta u+u=f(u) \quad \text{in } \mathbb{R}^3, \end{equation} where $f$ satisfies the following conditions: \begin{itemize} \item[(A2')] $f\in C(\mathbb{R},\mathbb{R})$, and $|f(u)|\leq C(1+|u|^{p-1})$ for some $C>0$ and $p\in (2,6)$; \item[(A3')] $f(u)= o(u)$ as $u\to 0$; \item[(A4')] there exists $\mu >3$ such that $f(u)u\geq \mu F(u)>0$ for all $u\in \mathbb{R}\backslash\{0\}$, where $F(u)=\int_0^uf(s)ds$. \end{itemize} \begin{theorem} \label{thm13} Suppose {\rm (A2')--(A4')} are satisfied, then \eqref{e1.6} has a ground state solution in $H^1(\mathbb{R}^3)$. \end{theorem} \begin{remark} \label{rmk1.5} \rm We note that in \cite{HZ1,HZ2,JW,WTXZ}, the nonlinearity $f$ is assumed to satisfy the Ambrosetti-Rabinowitz type 4-superlinear condition: \begin{itemize} \item[(AR)] there exists some $\mu>4$ such that $$ f(x,t)t\geq\mu F(x,t),~~\forall (x,t)\in\mathbb{R}^3\times\mathbb{R}. $$ \end{itemize} This condition plays a crucial role in obtaining the boundedness of the (PS) sequence of the functional $I$. Without condition (AR), it is difficult to get a bounded (PS) sequence and more techniques are involved. To overcome the difficulty, we use Jeanjean's monotonicity trick \cite{J} to construct a special (PS) sequence. For more applications about the monotonicity tricks and symmetry in variational principles, we refer the readers to Squassina’s papers \cite{Sq1} \cite{Sq2}. Using Pohozaev identity and a global compactness lemma, we can obtain that the special (PS) sequence is bounded and hence, we can obtain a nontrivial critical point. \end{remark} The article is organized as follows. In Section 2 we prove Theorem \ref{thm11} by using Szulkin and Weth's generalized Nehari manifold method. In Section 3 we present some estimates for the minimax level and give a threshold value (see Lemma \ref{lem3.3} below) under which the $(PS)_c$ condition is satisfied, and Theorem \ref{thm12} is proved. Section 4 is devoted to deal with the proof of Theorem \ref{thm13}. \smallskip \noindent\textbf{Notation.} Throughout this paper we shall denote by $C$, $c_i, C_i, i=1,2,\dots$ various positive constants whose exact value may change from lines to lines but are not essential to the analysis of problem. We will write $o(1)$ to denote quantity that tends to 0 as $n\to \infty$. For notational simplicity, we omit the integral symbol $dx$ in the integral representations below. Denote by $\mathbb{R}^+=[0,\infty)$. $ B_R(x)$ is the ball centered at the point $x$ with radius $R$. \section{Proof of Theorem \ref{thm11}} The main ingredient for the proof of Theorem \ref{thm11} is based on Szulkin and Weth's generalized Nehari manifold methods \cite{SW1}. From now on, we assume that (A1)--(A5) are satisfied. First, by (A2) and (A3), for any $\varepsilon >0$ there exists $C_\varepsilon >0$ such that \begin{equation} \label{e2.1} |f(x,u)|\leq \varepsilon|u|+C_\varepsilon |u|^{p-1} \quad \forall (x,u)\in (\mathbb{R}^3\times\mathbb{R}). \end{equation} By (A3) and (A5), one can easily check that \begin{equation} \label{e2.2} F(x,u) \geq0 \quad\text{and} \quad f(x,u)u>4F(x,u)>0\quad \text{if } u\neq0. \end{equation} Now we summarize some properties of $I$ on $ \mathcal {N}$ which are useful to study our problem. \begin{lemma} \label{lem2.1} Assume that {\rm(A1)--(A5)} are satisfied, then the following conclusions hold: \begin{itemize} \item[(i)] For $u\in E\backslash \{0\}$, there exists a unique $t_u=t(u)>0$ such that $ m(u) :=t_uu\in \mathcal {N}$ and $I(m(u))=\max_{t>0}I(tu)$. \item[(ii)] There exists $\alpha_0>0$ such that $\|u\|\geq \alpha_0$ for all $u\in \mathcal {N}$. \item[(iii)] $I$ is bounded from below on $\mathcal {N}$ by a positive constant. \item[(iv)] $I$ is coercive on $\mathcal {N}$, i.e., $I(u)\to \infty,\; as\;\|u\|\to\infty, u\in \mathcal {N}$. \item[(v)] Suppose $\mathcal {V}\subset E \backslash\{0\}$ is a compact subset, then there exists $R>0$ such that $I\leq0$ on $\mathbb{R}^+ \mathcal {V} \backslash B_R(0)$. \end{itemize} \end{lemma} \begin{proof} (i) For $t>0$, we denote \begin{align*} h(t): &=I(tu)=\frac{t^2}{2}\int_{\mathbb{R}^3}(a|\nabla u|^2+V(x)u^2)+\frac{bt^4}{4}\Big(\int_{\mathbb{R}^3}|\nabla u|^2\Big)^2-\int_{\mathbb{R}^3}F(x,tu) \\ &=\frac{t^2}{2}\|u\|^2+\frac{bt^4}{4}\Big(\int_{\mathbb{R}^3}|\nabla u|^2\Big)^2-\int_{\mathbb{R}^3}F(x,tu). \end{align*} From \eqref{e2.1} and the Sobolev embeddings $E\hookrightarrow L^2(\mathbb{R}^3)$, $E\hookrightarrow L^p(\mathbb{R}^3)$, for $\varepsilon$ sufficiently small we obtain \[ h(t)\geq\frac{t^2}{2}\|u\|^2 -\frac{\varepsilon t^2}{2}\int_{\mathbb{R}^3}|u|^2dx -\frac{C_\varepsilon t^p}{p}\int_{\mathbb{R}^3}|u|^p \geq \frac{t^2}{4}\|u\|^2-C_1t^p\|u\|^p \] where the constant $C_1$ is independent of $t$. Since $u\neq 0$ and $p>4$, It is easy to see that $h(t)>0$, whenever $t>0$ is small enough. On the other hand, nothing that $|tu(x)|\to \infty$ as $t\to\infty$, if $u(x)\neq0$. It follows from (A4) and Fatou's lemma that $$ h(t)\leq\frac{t^2}{2}\|u\|^2+\frac{Ct^4}{4}\|u\|^4 -t^4\int_{\mathbb{R}^3}\frac{F(x,tu)}{(tu)^4}u^4\to -\infty \quad \text{as } t\to \infty. $$ Therefore, $\max_{t>0}h(t)$ is achieved at some $t_u=t(u)>0$ such that $h' (t_u)=0$ and $t_uu\in \mathcal {N}$. To show the uniqueness of $t_u$, suppose by contradiction that there exists $t _u' >0$ with $t_u' \neq t_u$ such that $h' (t_u')=0$. Then $$ \frac{\|u\|^2}{(t_u')^2}+b\Big(\int_{\mathbb{R}^3}|\nabla u|^2dx\Big)^2 =\int_{\mathbb{R}^3}\frac{f(x,t_u' u)}{(t_u' u)^3}u^4. $$ This together with $$ \frac{\|u\|^2}{(t_u)^2}+b\Big(\int_{\mathbb{R}^3}|\nabla u|^2dx\Big)^2 =\int_{\mathbb{R}^3}\frac{f(x,t_u u)}{(t_u u)^3}u^4 $$ implies $$ \Big(\frac{1}{(t_u')^2}-\frac{1}{(t_u)^2} \Big)\|u\|^2 =\int_{\mathbb{R}^3}\left(\frac{f(x,t_u' u)}{(t_u' u)^3}-\frac{f(x,t_uu)}{(t_uu)^3} \right)u^4, $$ which contradicts (A5). (ii) Let $u\in \mathcal {N}$, by \eqref{e2.1}, for $\varepsilon$ small enough, we have \begin{align*} 0= \|u\|^2+b\Big(\int_{\mathbb{R}^3}|\nabla u|^2\Big)^2 -\int_{\mathbb{R}^3}f(x,u)u &\geq\|u\|^2-\varepsilon\int_{\mathbb{R}^3}|u|^2 -C_\varepsilon\int_{\mathbb{R}^3}|u|^p \\ &\geq \frac{1}{2}\|u\|^2-C_1\|u\|^p \end{align*} which implies $\|u\|\geq \alpha_0>0$ for all $u\in \mathcal {N}$. (iii) For $u\in \mathcal {N}$, it follows from (i) and \eqref{e2.2} that \begin{align*} I(u) &=I(u)-\frac{1}{4}\langle I'(u),u\rangle \\ &=\frac{1}{4}\|u\|^2+\int_{\mathbb{R}^3}\Big(\frac{1}{4}f(x,u)u-F(x,u)\Big) \\ &\geq \frac{1}{4}\|u\|^2\geq C_2>0. \end{align*} (iv) For $u\in \mathcal {N}$, it follows from (iii) $$ I(u)\geq \frac{1}{4}\|u\|^2, $$ which implies that $I$ is coercive on $\mathcal {N}$. (v) Without loss of generality, we may assume that $\|u\|=1$ for every $u\in \mathcal {V}$. Arguing indirectly suppose that there exist $u_n\in \mathcal {V}$ and $v_n=t_nu_n$ such that $I(v_n)\geq 0$ for all $n$ and $t_n\to\infty$ as $n\to\infty$. Passing to a subsequence, there exists $u\in E$ with $\|u\|=1$, such that $u_n\to u$. Note that $|v_n(x)|\to\infty$ if $u(x)\neq0$. By (A4) and Fatou's lemma we have $$ \int_{\mathbb{R}^3}\frac{F(x,v_n)}{v_n^4}u_n^4\to\infty $$ which implies $$ 0\leq \frac{I(v_n)}{\|v_n\|^4}=\frac{1}{2\|v_n\|^2} +\frac{b\big(\int_{\mathbb{R}^3}|\nabla v_n|^2dx\big)^2}{4\|v_n\|^4} -\int_{\mathbb{R}^3}\frac{F(x,v_n)}{v_n^4}u_n^4 \to-\infty, $$ a contradiction. \end{proof} Now we define the unit sphere $S :=\{u\in E: \|u\|=1\}$ of $E$ and the mapping $S\mapsto\mathcal {N}, u\mapsto m(u)$. As in \cite[Lemma 2.8]{SW1}, we have from Lemma \ref{lem2.1} the following key observation: the mapping $m$ is continuous and moreover $m$ is homeomorphism between $S$ and $\mathcal {N}$, where the inverse of $m$ is given by \begin{equation} \label{e2.3} m^{-1}(u)=\frac{u}{\|u\|}. \end{equation} Now we consider the functional $\Psi :S\to \mathbb{R}$ defined by $\Psi(w) :=I(m(w)$. As in \cite[Prop. 2.9 and Cor. 2.10]{SW1}, the following lemma follows as a consequence of Lemma \ref{lem2.1} and the above observation. \begin{lemma} \label{lem2.2} \begin{itemize} \item[(i)] $\Psi(w)\in C^1(S,\mathbb{R})$, and $$ \Psi'(w)z=\|m(w)\|\langle I'(m(w)),z\rangle,\quad \text{for any } z\in T_wS=\{v\in E : \langle v,w\rangle=0\} . $$ \item[(ii)] $\{w_n\}$ is a Palais-Smale sequence for $\Psi$ if and only if $\{m(w_n)\}$ is a Palais-Smale sequence for $I$. \item[(iii)] $w\in S$ is a critical point of $\Psi$ if and only if $m(w)\in \mathcal {N}$ is a critical point of $I$. Moreover, the corresponding critical values of $\Psi$ and $I$ coincide and $\inf_S\Psi=\inf_\mathcal {N}I$. \item[(iv)] If $I$ is even, then so is $\Psi$. \end{itemize} \end{lemma} Now we set the infimum of $I$ on $\mathcal {N}$ by $$ c=\inf_{\mathcal {N}}I=\inf_{S}\Psi. $$ We recall the following result due to P.L. Lions \cite[Lemma 1.21]{W}). \begin{lemma} \label{lem2.3} Let $r>0$, If $\{u_n\}$ is bounded in $H^1(\mathbb{R}^3)$ and $$ \lim_{n\to\infty} \sup_{y\in \mathbb{R}^3}\int_{B_r(y)}|u_n|^2=0, $$ then $u_n\to0 $ in $L^s(\mathbb{R}^3)$ for any $s\in (2,6)$. \end{lemma} Now we are ready to study the minimizing sequence for $I$ on $\mathcal {N}$. \begin{lemma} \label{lem2.4} Let $\{u_n\}\subset \mathcal {N}$ be a minimizing sequence for $I$. Then $\{u_n\}$ is bounded. Moreover, after a suitable $\mathbb{Z}^3$-translation, passing to a subsequence there exists $u\in \mathcal {N}$ such that $u_n\rightharpoonup u$ and $I(u)=\inf_\mathcal {N} I$. \end{lemma} \begin{proof} Let $\{u_n\}\subset \mathcal {N}$ be a minimizing sequence such that $I(u_n)\to c$. Then $\{u_n\}$ is bounded by Lemma \ref{lem2.1} (iv). Therefore $u_n\rightharpoonup u$ for some $u\in E$, after passing to a subsequence. Assume that \begin{equation} \label{e2.4} \lim_{n\to\infty}\sup_{y\in \mathbb{R}^3}\int_{B_r(y)}|u_n|^2=0, \end{equation} then from Lemma \ref{lem2.3} we conclude that $u_n\to0$ in $L^s(\mathbb{R}^3)$ for any $s\in(2,6)$ and so it is standard to show that $\int_{\mathbb{R}^3}f(x,u_n)u_n=o(1) $ as $n\to\infty$ by \eqref{e2.1}. Therefore, $$ 0=\langle I'(u_n),u_n\rangle=\|u_n\|^2+b\Big(\int_{\mathbb{R}^3}|\nabla u_n|^2\Big)^2-\int_{\mathbb{R}^3}f(x,u_n)u_n \geq \|u_n\|^2-o(1), $$ which implies $\|u_n\|\to0$, contrary to Lemma \ref{lem2.1} (ii). Hence \eqref{e2.4} cannot hold, and so, there exist $r,\delta>0$ and a sequence $\{y_n\}\subset \mathbb{R}^3$ such that $$ \lim_{n\to\infty}\int_{B_r(y_n)}|u_n|^2\geq\delta>0. $$ Here we may assume $y_n\in\mathbb{Z}^3$ by taking a larger $r$ if necessary. In view of $I$ and $\mathcal {N}$ are invariant under translations, we may assume that $\{y_n\}$ is bounded in $\mathbb{Z}^3$. Thus, passing to a subsequence we have $u_n\rightharpoonup u\neq0$. Now we prove that $u$ is a critical point of $I$. Indeed, since $\{u_n\}$ is bounded, then up to a subsequence, $u_n\to u$ in $L^p_{loc}(\mathbb{R}^3), p\in [1,6), u_n\to u$ a.e. in $\mathbb{R}^3$, and we may suppose $$ \int_{\mathbb{R}^3}|\nabla u_n|^2\to A^2\geq0. $$ For any $\phi\in C^{\infty}_0(\mathbb{R}^3)$, we have $I'(u_n)\phi=o(1)$. That is \begin{equation} \label{e2.5} I'(u_n)\phi=\int_{\mathbb{R}^3}(a\nabla u_n\nabla\phi+Vu_n\phi)+b\int_{\mathbb{R}^3}|\nabla u_n|^2\int_{\mathbb{R}^3}\nabla u_n\nabla\phi-\int_{\mathbb{R}^3}f(x,u_n)\phi=o(1). \end{equation} Passing to a limit as $n\to\infty$, we have \begin{equation} \label{e2.6} 0=\int_{\mathbb{R}^3}(a\nabla u\nabla\phi+Vu\phi)+bA^2\int_{\mathbb{R}^3}\nabla u\nabla\phi-\int_{\mathbb{R}^3}f(x,u)\phi, \end{equation} for any $\phi\in C^{\infty}_0(\mathbb{R}^3)$. By Lemma \ref{lem2.1} (iii) we know that $c>0$, and so $A>0$. Next we show that $$ \int_{\mathbb{R}^3}|\nabla u|^2=A^2. $$ Notice that $$ A^2=\liminf_{n\to\infty}\int_{\mathbb{R}^3}|\nabla u_n|^2\geq\int_{\mathbb{R}^3}|\nabla u|^2. $$ Suppose by contradiction, that $$ \int_{\mathbb{R}^3}|\nabla u|^20$ for some $0<\theta_0<<1$. Thus, there is $\theta\in(\theta_0,1)$ such that $I'(\theta u)\theta u=0$. Consequently, by Fatou's lemma, we conclude that \begin{align*} c&\leq I(\theta u)=I(\theta u)-\frac{1}{4}I'(\theta u)\theta u \\ &= \frac{\theta^2}{4}\int_{\mathbb{R}^3}(a|\nabla u|^2+Vu^2)-\int_{\mathbb{R}^3}\big(\frac{1}{4}f(x,u)u-F(x,u)\big) \\ &\leq \liminf_{n\to\infty}\Big\{\frac{\theta^2}{4}\int_{\mathbb{R}^3}(a|\nabla u_n|^2+Vu^2_n)-\int_{\mathbb{R}^3}\Big(\frac{1}{4}f(x,\theta u_n) \theta u_n-F(x,\theta u_n)\Big)\Big\} \\ &< \liminf_{n\to\infty}\Big\{\frac{1}{4}\int_{\mathbb{R}^3}(a|\nabla u_n|^2+Vu^2_n)-\int_{\mathbb{R}^3}\Big(\frac{1}{4}f(x,u_n)u_n-F(x,u_n)\Big)\Big\} \\ &= \liminf_{n\to\infty}\big\{I(u)-\frac{1}{4}I'(u)u\big\} = c, \end{align*} which leads to a contradiction. Here we have used the following facts: \begin{itemize} \item[(i)] $\frac{1}{4}f(x,t)t\geq F(x,t)\geq0$ for all $t\in\mathbb{R}$. \item[(ii)] $\frac{1}{4}f(x,t)t-F(x,t)$ is nondecreasing for $t\geq0;$ and nonincreasing for $t\leq0$. \end{itemize} In fact, property (i) can be easily checked by using (A3), (A5). We only check property (ii) for the case $t\leq0$. Indeed, letting $s\frac{1}{4}f(x,s)s-F(x,t)+\int_s^t\frac{f(x,s)}{s^3}\tau^3d\tau \\ &= \frac{1}{4}f(x,s)s-F(x,t)+\frac{f(x,s)}{s^3}\frac{1}{4}[t^4-s^4] \\ &= \frac{t^4}{4}\frac{f(x,s)}{s^3}-F(x,t) \\ &>\frac{t^4}{4}\frac{f(x,t)}{t^3}-F(x,t) \\ &= \frac{1}{4}f(x,t)t-F(x,t). \end{align*} The above contradiction shows that \begin{equation} \label{e2.7} \int_{\mathbb{R}^3}|\nabla u_n|^2\to A^2=\int_{\mathbb{R}^3}|\nabla u|^2. \end{equation} Hence, from \eqref{e2.5}-\eqref{e2.6} we have that $I'(u)=0$. So, $u\in \mathcal {N}$. Clearly, $I(u)\geq c$. To complete the proof, it remains to prove that $I(u)\leq c$. In fact, from \eqref{e2.2}, Fatou's lemma, the weakly lower semi-continuity of $\|\cdot\|$ and the boundedness of $\{u_n\}$, we obtain \begin{align*} c+o(1)&=I(u_n)-\frac{1}{4}\langle I'(u_n),u_n\rangle\\ &=\frac{1}{4}\|u_n\|^2+\int_{\mathbb{R}^3} \Big(\frac{1}{4}f(x,u_n)u_n-F(x,u_n)\Big)\\ &\geq \frac{1}{4}\|u\|^2+\int_{\mathbb{R}^3} \Big(\frac{1}{4}f(x,u)u-F(x,u)\Big)+o(1) \\ &=I(u)-\frac{1}{4}\langle I'(u),u\rangle+o(1) \\ &=I(u)+o(1) \end{align*} which implies $I(u)\leq c$. The proof is completed. \end{proof} Now we are ready to prove the existence and compactness of Theorem \ref{thm11}. \begin{proof}[Proof of Theorem \ref{thm11}] (i) Let $c=\inf_\mathcal {N} I$ as mentioned above. From Lemma \ref{lem2.1} (iii) we see that $c>0$. Moreover, if $u_0\in \mathcal {N}$ satisfies $I(u_0)=c$, then $m^{-1}(u_0)\in S$ is a minimizer of $\Psi$ and therefore a critical point of $\Psi$. Thus by Lemma \ref{lem2.2} (iii) $u_0$ is a critical point of $I$. It remains to prove that there exists a minimizer $u$ of $I|_{\mathcal {N}}$. By Ekeland's variational principle \cite{W}, there exists a sequence $\{w_n\}\subset S$ with $\Psi(w_n)\to c$ and $\Psi'(w_n)\to 0$ as $n\to \infty$. Put $u_n=m(w_n)$. Then from Lemma \ref{lem2.2}(ii), we have that $I(u_n)\to c$ and $I'(u_n)\to 0$ as $n\to \infty$. Consequently, $\{u_n\}$ is a minimizing sequence for $I$ on $\mathcal {N}$. Therefore, by Lemma \ref{lem2.4} exists a minimizer $u$ of $I|_{\mathcal {N}}$, as required. (ii) Let $\{u_n\}\subset \mathcal {K}$ be a bounded sequence. Then $u_n\in \mathcal {N}, I(u_n)=c$ and $I' (u_n)=0$. Up to a subsequence, we may assume $u_n\rightharpoonup u$ in $E$. From Lemma \ref{lem2.4}, we have that $\{u_n\}$ is non-vanishing, i.e., $$ \lim_{n\to\infty}\int_{B_r(y_n)}|u_n|^2\geq\delta>0. $$ By the invariance of $I$ on $\mathcal {N}$ under the translations of the form $u\mapsto u(\cdot -k)$ with $k\in \mathbb{Z}^3$, we may assume that $\{y_n\}$ is bounded in $\mathbb{Z}^3$. Therefore $u_n\rightharpoonup u\neq 0$ and $I'(u)=0$. Again by Lemma \ref{lem2.4}, one obtains that $I(u)=c$. So we obtain \begin{equation} \label{e2.8} \begin{aligned} c&=I(u)-\frac{1}{4}\langle I'(u),u\rangle \\ &=\frac{1}{4}\|u\|^2+\int_{\mathbb{R}^3}\Big(\frac{1}{4}f(x,u)u-F(x,u)\Big) \\ &\leq\lim_{n\to \infty}\Big(\frac{1}{4}\|u_n\|^2 +\int_{\mathbb{R}^3}\Big(\frac{1}{4}f(x,u_n)u_n-F(x,u_n)\Big)\Big) \\ &=\lim_{n\to \infty}\Big(I(u_n)-\frac{1}{4}\langle I'(u_n),u_n\rangle\Big)=c \end{aligned} \end{equation} which implies that$\|u_n\|\to\|u\|$. Hence $u_n\to u$ in $E$. \end{proof} \section{Proof of Theorem \ref{thm12}} In this section we always assume that (A1), (A6)--(A9) are satisfied. Note that (A8) and (A9) imply $g(x,u)=O(u^3)$ as $u\to 0$ and \begin{equation} \label{e3.1} g(x,u)u\geq 4 G(x,u)\geq0. \end{equation} Moreover, by (A7) and (A8), for any $\varepsilon>0$ there exists $C_\varepsilon>0$ such that \begin{equation} \label{e3.2} |g(x,u)|\leq\varepsilon|u|+C_\varepsilon|u|^{p-1}, \quad \forall (u,x)\in \mathbb{R}\times\mathbb{R}^3. \end{equation} We denote the energy functional associated with \eqref{e1.3} by \[ J(u)=\frac{1}{2}\int_{\mathbb{R}^3} (a|\nabla u|^2+V(x)u^2)+\frac{b}{4}\Big(\int_{\mathbb{R}^3} |\nabla u|^2dx\Big)^2-\lambda\int_{\mathbb{R}^3} G(x,u)-\frac{1}{6}\int_{\mathbb{R}^3}|u|^6, \] for all $u\in E$. Next we enunciate without proof the following lemma, the proof follows from a similar argument to that used in the proof of Lemma \ref{lem2.2}. The Nehari manifold for \eqref{e1.5} is still denoted by $\mathcal {N}$. \begin{lemma} \label{lem3.1} \begin{itemize} \item[(i)] For $u\in E\backslash \{0\}$, there exists a unique $t_u=t(u)>0$ such that $m(u) :=t_uu\in \mathcal {N}$ and $I(m(u))=\max I(\mathbb{R}^+u)$. \item[(ii)] There exists $\alpha_0>0$ such that $\|u\|\geq\alpha_0$ for all $u\in\mathcal {N}$. \item[(iii)] $J$ is bounded from below on $\mathcal {N}$ by a positive constant. \item[(iv)] $J$ is coercive on $\mathcal {N}$. \item[(v)] Suppose $\mathcal {V}\in E\backslash \{0\}$ is a compact subset, then there exists $R>0$ such that $J\leq0$ on $\mathbb{R}^+\mathcal {V}\backslash B_R(0)$. \end{itemize} \end{lemma} From Lemma \ref{lem3.1}, and arguing as \cite[Lemma 2.8]{SW1}, we see that the mapping $m$ is continuous. $m$ is homeomorphism between $S$ and $\mathcal {N}$, and the inverse of $m$ is given by $m^{-1}(u)=\frac{u}{\|u\|}$. Now we consider the functional $\Psi :S\to\mathbb{R}$ defined by \begin{align*} \Psi(w):=J(m(w)). \end{align*} Similar to Lemma \ref{lem2.2}, we have the following parallel lemma. \begin{lemma} \label{lem3.2} \begin{itemize} \item[(i)] $\Psi\in C^1(S,\mathbb{R})$, and $$ \Psi'(w)z=\|m(w)\|\langle J'(m(w)),z \rangle\quad \text{for any } z\in T_wS=\{v\in E : \langle v,w\rangle=0\} $$ \item[(ii)] $\{w_n\}$ is a Palais-Smale sequence for $\Psi$ if and only if $\{m(w_n)\}$ is a Palais-smale sequence for $J$. \item[(iii)] $w\in S$ is a critical point of $\Psi$ if and only if $m(w)\in\mathcal {N}$ is a critical point of $J$. Moreover, the corresponding critical values of $\Psi$ and $J$ coincide and $\inf_S\Psi=\inf_\mathcal {N}J$. \end{itemize} \end{lemma} Recall that $c=\inf_\mathcal {N}J$. By Lemma \ref{lem3.1} (iii), $c>0$. Applying Ekeland's variational principle, there exists a Palais-Smale sequence $\{w_n\}\subset S$ for $\Psi$ such that $\Psi(w_n)\to c$. Set $u=m(w_n)$. Then from Lemma \ref{lem3.2} (ii), $\{u_n\}\subset\mathcal {N}$ is a Palais-Smale sequence for $J$ and $J(u_n)\to c$. By Lemma \ref{lem3.1} (iv), $ \{u_n\}$ is bounded. Then $\{u_n\}$ is either (i) Vanishing: for each $r>0$, $$ \lim_{n\to\infty}\sup_{y\in \mathbb{R}^3}\int_{B_r(y)}|u_n|^2=0; $$ or (ii) Non-vanishing: there exists $r,\delta>0$ and a sequence $\{y_n\}\subset\mathbb{R}^3$ such that $$ \lim_{n\to\infty}\int_{B_r(y_n)}|u_n|^2\geq\delta. $$ In case (ii) we may assume $y_n\in \mathbb{Z}^3$ by taking a larger $r$ if necessary. Suppose case (ii) holds and let $\widetilde{u}_n(x) :=u_n(x+y_n)$. Since $J$ is invariant and $\nabla J$ is equivariant with respect to the $\mathbb{Z}^3$-action, $\widetilde{u}_n\rightharpoonup u$ up to a subsequence, $J'(u)=0, J(u)\geq c$ and since $\lim_{n\to\infty}\int_{B_r(y)}|u_n|^2\geq\delta$, $u\neq0$. Hence $u$ is a nontrivial critical point of $J$. Moreover, $u\in\mathcal {N}$ and $J(u)\geq c$. Consequently, $J(u)=c$ and thus $u$ is a ground state solutions of problem \eqref{e1.5}. It remains to prove that vanishing cannot occur. This will be done in the following two lemmas. We assume without loss of generality that, $K(x_0)=K(0)=\max_{x\in\mathbb{R}^3}K(x)=:\|K\|_\infty$. We note that the critical equation \begin{equation} \label{e3.3} -\Delta u=u^5\quad\text{in }\mathbb{R}^3 \end{equation} has the well known minimal decaying positive solution $$ u=u_{\varepsilon}=K\Big(\frac{\varepsilon}{\varepsilon^2+|x|^2}\Big)^{1/2}, \quad K=3^{1/4} $$ for any $\varepsilon>0$. It is well known that \cite{BN}, $u_{\varepsilon}$ satisfies \begin{equation} \label{ma-he023} \int_{\mathbb{R}^3}|\nabla u_{\varepsilon}|^2 =\int_{\mathbb{R}^3}|u_{\varepsilon}|^6=S^{3/2}, \end{equation} where $S$ is the best Sobolev embedding constant given by \begin{equation} \label{e3.5} S :=\inf_{u\in D^{1,2}(\mathbb{R}^3)\backslash\{0\}} \frac{\int_{\mathbb{R}^3}|\nabla u|^2} {(\int_{\mathbb{R}^3}|u|^6)^{1/3}}. \end{equation} Define \[ w_{\varepsilon}(x)=\eta(x)u_{\varepsilon}(x),\quad x\in\mathbb{R}^3, \varepsilon>0, \] where $\eta\in C_0^\infty(\mathbb{R}^3,[0,1])$ is a piecewise smooth function with support in $B_{2R}(0)$ such that $\eta(x)=1$ in $B_R(0), 0\leq\eta(x)\leq1$ in $B_{2R}(0)$ and $|\nabla \eta|\leq C/R$. As in \cite{BN,NSY}, we have the following estimates as $\varepsilon\to0^+$. \begin{gather} \label{e3.6} \|\nabla w_\varepsilon\|_2^2=S^{3/2}+O(\varepsilon),\quad \|w_\varepsilon\|_6^6=S^{3/2}+O(\varepsilon^3) , \\ \label{e3.7} \|w_\varepsilon\|_s^s=\begin{cases} O(\varepsilon^{s/2}),& \text{if } s\in [2,3),\\ O(\varepsilon^{s/2}|\ln \varepsilon|),& \text{if }s=3, \\ O(\varepsilon^\frac{6-s}{2}),& \text{if } s\in (3,6). \end{cases} \end{gather} Since $K(x)-K(0)=O(|x|^\alpha)$ as $x\to0$, as in \cite[Lemma 2]{GL}, by \eqref{e3.6}, as $\varepsilon\to0^+$ we have \begin{equation} \label{e3.8} \begin{aligned} \int_{\mathbb{R}^3}K(x)|w_\varepsilon|^6 &= \|K\|_\infty\int_{\mathbb{R}^3}|w_\varepsilon|^6+\int_{\mathbb{R}^3}(K(x)-K(0))|w _\varepsilon|^6 \\ &= \|K\|_\infty S^{3/2}+O(\theta(\varepsilon)), \end{aligned} \end{equation} where \begin{equation} \label{e3.9} \theta(\varepsilon)=\begin{cases} \varepsilon^\alpha, &\text{if }\alpha<3\\ \varepsilon^3|\ln\varepsilon|, &\text{if } \alpha=3 \\ \varepsilon^3, &\text{if }\alpha>3. \end{cases} \end{equation} Let \begin{equation} \label{e3.10} v_{\varepsilon}(x)=w_{\varepsilon} \Big[\int_{\mathbb{R}^3}K(x)|w_{\varepsilon}|^6\Big]^{-1/6}. \end{equation} \begin{lemma} \label{lem3.3} \[ c0$ the integral $\int_{B_R(0)}K(x)|w_{\varepsilon}|^6$ is bounded below by a positive constant, independent of $\varepsilon$. Hence, \eqref{ma-he023} and \eqref{ma-he023b} imply the inequality \begin{equation} \label{e3.15} W_{\varepsilon}=\int_{\mathbb{R}^3}|\nabla v_{\varepsilon}|^2\leq S\|K\|_{\infty}^{-1/3}+O(\varepsilon^{\beta}). \end{equation} where $\beta=\min\{\alpha,1\}$. By Lemma \ref{lem3.1} (i) and (iii), there exists $t_\varepsilon>0$ such that $$ J(t_\varepsilon v_\varepsilon)=\max_{t\geq0}J(tv_\varepsilon)\geq C. $$ From the continuity of $J$, we see that there exists $t_0>0$ independent of $\varepsilon$ satisfying $t_\varepsilon>t_0>0$. Put $$ \zeta(t)=\frac{t^2}{2}\int_{\mathbb{R}^3} (a|\nabla v_\varepsilon|^2+V(x)v_\varepsilon^2)+\frac{bt^4}{4}\Big(\int_{\mathbb{R}^3} |\nabla v_\varepsilon|^2dx\Big)^2-\frac{t^6}{6}. $$ Then it is easy to see that $\zeta(t)$ achieves its maximum at the global maximum point $\tilde{t}_{\varepsilon}>0$, satisfying $$ \int_{\mathbb{R}^3}(a|\nabla v_\varepsilon|^2+V(x)v^2_{\varepsilon}) +b(\tilde{t}_{\varepsilon})^2\Big(\int_{\mathbb{R}^3} |\nabla v_\varepsilon|^2\Big)^2- (\tilde{t}_{\varepsilon})^{4}=0. $$ Then $\tilde{t}_{\varepsilon}$ takes the form \begin{equation} \label{e3.16} (\tilde{t}_{\varepsilon})^2=\frac{bW^2_{\varepsilon}+\sqrt{b^2W^4_{\varepsilon} +4(aW_{\varepsilon}+\int_{\mathbb{R}^3}V(x)v^2_{\varepsilon})}}{2}:=T_0. \end{equation} As in \cite{HLP}, denote $c_1=bW_{\varepsilon}^2$, \[ c_2=aW_{\varepsilon}+\int_{\mathbb{R}^3}V(x)v_{\varepsilon}^2. \] Using (A9), $t_{\varepsilon}>t_0$, we obtain \begin{equation} \label{e3.17} \begin{aligned} J(t_\varepsilon v_\varepsilon) &= \zeta(t_{\varepsilon})- \lambda\int_{\mathbb{R}^3} G(x,t_{\varepsilon}v_{\varepsilon}) \\ &\leq \zeta(\tilde{t}_{\varepsilon})-\lambda c_0t^q_{\varepsilon}\int_{\mathbb{R}^3} |v_{\varepsilon}|^q \\ &\leq \frac{(\tilde{t}_{\varepsilon})^2}{2}\int_{\mathbb{R}^3} \left(a|\nabla v_{\varepsilon}|^2+V(x)v_{\varepsilon}^2\right) +\frac{b(\tilde{t}_{\varepsilon})^4}{4}\Big(\int_{\mathbb{R}^3} |\nabla v_{\varepsilon}|^2\Big)^2 \\ &\quad -\frac{(\tilde{t}_{\varepsilon})^6}{6}-\lambda C_1\int_{\mathbb{R}^3} |v_{\varepsilon}|^q \\ &= \frac{T_0}{2}\int_{\mathbb{R}^3} \big(a|\nabla v_{\varepsilon}|^2+V(x)v_{\varepsilon}^2\big) +\frac{bT^2_0}{4}\Big(\int_{\mathbb{R}^3} |\nabla v_{\varepsilon}|^2\Big)^2 \\ &\quad -\frac{T^3_0}{6}-\lambda C_1\int_{\mathbb{R}^3} |v_{\varepsilon}|^q \\ &= \frac{T_0}{2}\Big(aW_{\varepsilon}+\int_{\mathbb{R}^3}V(x) v_{\varepsilon}^2\Big)+\frac{1}{4}{bT^2_0W^2_{\varepsilon}} -\frac{1}{6}{T^3_0}-\lambda C_1\int_{\mathbb{R}^3} |v_{\varepsilon}|^q \\ &= \frac{1}{24}(c_1+c_2)^{3/2}+\frac{1}{24}c_1c_2+\frac{1}{24}c_1^3-\lambda C_1\int_{\mathbb{R}^3} |v_{\varepsilon}|^q. \end{aligned} \end{equation} Using \eqref{e3.15} and inequality \begin{equation} \label{e3.18} (a+b)^p\leq a^p+p(a+b)^{p-1}b,\quad p\geq1,\; ab>0 \end{equation} we conclude that \begin{equation} \label{e3.19} \begin{aligned} J(t_\varepsilon v_\varepsilon) &\leq \frac{1}{24}(b^2W_{\varepsilon}^4+4aW_{\varepsilon})^{3/2} +C_1\int_{\mathbb{R}^3}V(x)v^2_{\varepsilon} +\frac{1}{4}abW^3_{\varepsilon}+C_2\int_{\mathbb{R}^3}V(x)v^2_{\varepsilon} \\ &\quad +\frac{1}{24}b^3W^3_{\varepsilon}-\lambda C_1\int_{\mathbb{R}^3} |v_{\varepsilon}|^q \\ &\leq\frac{1}{24} \big[b^2[S\|K\|_{\infty}^{-1/3}+O(\varepsilon^{\beta})]^4 +4a[S\|K\|_{\infty}^{-1/3}+O(\varepsilon^{\beta})]\big]^{3/2} \\ &\quad +\frac{1}{4}ab[S\|K\|_{\infty}^{-1/3}+O(\varepsilon^{\beta})]^3 +\frac{1}{24}b^3[S\|K\|_{\infty}^{-1/3}+O(\varepsilon^{\beta})]^3 \\ &\quad +C_3\int_{\mathbb{R}^3}V(x)v^2_{\varepsilon}-\lambda C_1\int_{\mathbb{R}^3} |v_{\varepsilon}|^q \\ &\leq \frac{1}{24}\left[b^2S^4\|K\|_{\infty}^{-4/3} +4aS\|K\|_{\infty}^{-1/3}\right]^{3/2}+ \frac{1}{4}abS^3\|K\|_{\infty}^{-1}+\frac{1}{24}b^3S^6\|K\|_{\infty}^{-2} \\ &\quad +C_3\int_{\mathbb{R}^3}V(x)v^2_{\varepsilon}+O(\varepsilon^{\beta})-\lambda C_1\int_{\mathbb{R}^3} |v_{\varepsilon}|^q, \end{aligned} \end{equation} where $C_i$, $i=1,2,3$, are positive constants, independent of $\varepsilon$. If $\alpha\geq1$ then $\beta=1$, Hence to complete the proof, it remains to show that \begin{gather} \label{e3.20} \lim_{\varepsilon\to0^+}\frac{1}{\varepsilon}\int_{B_R(0)}\left[C_3V(x) v_\varepsilon^2-\lambda C_1 |v_\varepsilon|^q\right]=-\infty, \\ \label{e3.21} \lim_{\varepsilon\to0^+}\frac{1}{\varepsilon}\int_{\mathbb{R}^3\backslash B_R(0)}\left[C_3V(x) v_\varepsilon^2-\lambda C_1 |v_\varepsilon|^q\right]\leq C_4. \end{gather} In fact, from \eqref{e3.7}, \eqref{e3.8} it is easy to see that \begin{gather} \label{e3.22} \frac{1}{\varepsilon}\int_{B_R(0)}C_3V(x) v_\varepsilon^2\leq\frac{C}{\varepsilon} \int_{B_R(0)}\frac{\varepsilon}{\varepsilon^2+|x|^2}\leq C_R, \\ \label{e3.23} \frac{\lambda C_1}{\varepsilon}\int_{B_R(0)}|v_{\varepsilon}|^q \geq\frac{\lambda C}{\varepsilon}\int_{B_R(0)}|w_{\varepsilon}|^q =\frac{\lambda C}{\varepsilon}\int_{B_R(0)} \frac{\varepsilon^{\frac{q}{2}}}{(\varepsilon^2+|x|^2)^{\frac{q}{2}}} \geq \lambda C\varepsilon^{\frac{-q+4}{4}}. \end{gather} Again by \eqref{e3.6}, we have \begin{equation}\label{e3.24} \begin{aligned} \frac{1}{\varepsilon} \int_{\mathbb{R}^3\backslash B_R(0)} \big[C_3V(x) v_\varepsilon^2-\lambda C_1 |v_\varepsilon|^q\big] &\leq \frac{1}{\varepsilon}\int_{B_{2R}(0)\backslash B_R(0)}C_3V(x) v_{\varepsilon}^2 \\ &\leq \frac{1}{\varepsilon}\int_{B_{2R}(x_0)\backslash B_R(0)}C_6 w_{\varepsilon}^2 \leq C_7, \end{aligned} \end{equation} where $C_i$, $i=4,5,6,7$, are positive constants, independent of $\varepsilon$. If $40$. If $q=4$, one can chose $\lambda=\varepsilon^{-\delta}$, $\delta>0$ in inequality \eqref{e3.23} to obtain \eqref{e3.20}. If $0<\alpha<1$, then $\beta=\alpha$. Choosing $\varepsilon$ so small that $B_\varepsilon(0)\subset B_R(0) \subset \Omega$, then by \eqref{e3.1} we have $$ \int_{\mathbb{R}^3}G(x,t_\varepsilon v_\varepsilon)\geq\int_{B_\varepsilon(0)}G \Big(x,t_\varepsilon w_{\varepsilon} \Big[\int_{\mathbb{R}^3}K(x)|w_{\varepsilon}|^6\Big]^{-1/6} \Big). $$ Since $t_\varepsilon\geq t_0$, by \eqref{e3.7} and the definition of $w_{\varepsilon}$ it is easy to check that, for all $x\in B_\varepsilon(0)$, \begin{align*} t_\varepsilon w_{\varepsilon} \Big[\int_{\mathbb{R}^3}K(x)|w_{\varepsilon}|^6\Big]^{-\frac{1}{6}} &=\frac{3^\frac{1}{4} t_\varepsilon \varepsilon^{1/2}} {(\varepsilon^2+|x|^2)^{1/2}}\times\Big(\|K\|_\infty S^{3/2}+O(\varepsilon^{\alpha})\Big)^{-1/6}\\ &\geq Ct_0\varepsilon^{-1/2}\to+\infty \end{align*} as $\varepsilon\to0^+$, which jointly with $(g_4')$, implies that for any $M>0$ there exists $\varepsilon_0>0$ such that for all $\varepsilon\in (0,\varepsilon_0)$ \begin{equation} \label{e3.25} \int_{\mathbb{R}^3}G(x,t_\varepsilon v_\varepsilon) \geq C_1M\int_{B_\varepsilon(0)}\varepsilon^{\alpha-3}=C_2M\varepsilon^\alpha. \end{equation} It follows from \eqref{e3.17}, \eqref{e3.19} and \eqref{e3.25} that \begin{equation} \label{e3.26} \begin{aligned} J(t_\varepsilon v_\varepsilon) &\leq \frac{1}{24}\left[b^2S^4\|K\|_{\infty}^{-4/3} +4aS\|K\|_{\infty}^{-1/3}\right]^{3/2} + \frac{1}{4}abS^3\|K\|_{\infty}^{-1}+\frac{1}{24}b^3S^6\|K\|_{\infty}^{-2} \\ &\quad +O(\varepsilon^{\alpha})+C_1\varepsilon-C_2M\varepsilon^\alpha. \end{aligned} \end{equation} Hence taking $M$ large enough and for $\varepsilon$ small enough, we deduce that $$ J(t_\varepsilon v_\varepsilon) \leq \frac{1}{24}\left[b^2S^4\|K\|_{\infty}^{-4/3} +4aS\|K\|_{\infty}^{-1/3}\right]^{3/2}+ \frac{1}{4}abS^3\|K\|_{\infty}^{-1}+\frac{1}{24}b^3S^6\|K\|_{\infty}^{-2}, $$ as required. \end{proof} \begin{lemma} \label{lem3.4} If $c\in (0,c^*)$, then $\{u_n\}$ cannot vanish. \end{lemma} \begin{proof} Suppose by contradiction that $\{u_n\}$ is vanishing, then it follows Lemma \ref{lem2.3}, that $u_n\to0$ in $L^s(\mathbb{R}^3)$ whenever $20$, otherwise $\|u_n\|\to0$ as $n\to\infty$ which contradicts to $c>0$. By \eqref{e3.5} we have $$ \int_{\mathbb{R}^3}|\nabla u_n|^2 \geq S\Big(\int_{\mathbb{R}^3}|u_n|^6\Big)^{1/3} \geq S\|K\|_{\infty}^{-1/3} \Big(\int_{\mathbb{R}^3}K(x)|u_n|^6\Big)^{1/3}. $$ Then, we have \begin{gather*} \|u_n\|^2\geq a\int_{\mathbb{R}^3}|\nabla u_n|^2\geq a S\|K\|_{\infty}^{-1/3}\Big(\int_{\mathbb{R}^3}K(x)|u_n|^6\Big)^{1/3},\\ b\Big(\int_{\mathbb{R}^3}|\nabla u_n|^2\Big)^2 \geq bS^2\|K\|_{\infty}^{-2/3}\Big(\int_{\mathbb{R}^3}K(x)|u_n|^6\Big)^{2/3}. \end{gather*} Passing the limit in the previous two inequalities, as $n\to\infty$, we obtain \begin{gather*} l_1\geq aS\|K\|_\infty^{-1/3}(l_1+l_2)^{1/3}, \\ l_2\geq bS^2\|K\|_\infty^{-2/3}(l_1+l_2)^{2/3}. \end{gather*} Hence $$ (l_1+l_2)^{1/3}\geq \frac{\|K\|_\infty^{-2/3}(bS^2+\sqrt{b^2S^4+4aS\|K\|_\infty})}{2}. $$ Then \begin{align*} c&=\lim_{n\to\infty}J(u_n)\\ &=\frac{1}{3}l_1+\frac{1}{12}l_2 \\ &\geq\frac{1}{3}a S\|K\|_\infty^{-1/3}(l_1+l_2)^{1/3} +\frac{1}{12}bS^2\|K\|_\infty^{-2/3}(l_1+l_2)^\frac{2}{3} \\ &\geq\frac{1}{3}a S\|K\|_\infty^{-1}\frac{bS^2+\sqrt{b^2S^4+4aS\|K\|_{\infty}}}{2} \\ &\quad +\frac{1}{12}bS^2\|K\|_\infty^{-2} \frac{(bS^2+\sqrt{b^2S^4+4aS\|K\|_{\infty}})^2}{4} \\ &=\frac{abS^3\|K\|_\infty^{-1}}{4}+\frac{b^3S^6\|K\|_\infty^{-2}}{24} +\frac{(b^2S^4+4aS\|K\|_\infty)^{3/2}\|K\|_\infty^{-2}}{24} =: c^* \end{align*} which contradicts that $c0. $$ From the invariance of $J$ and $\mathcal {N}$ under the translations of the form $u\mapsto u(\cdot -k)$ with $k\in\mathbb{Z}^3$, we may assume that $\{y_n\}$ is bounded in $\mathbb{Z}^3$. Therefore, $u_n\rightharpoonup u\neq0$ and $J'(u_n)=0$. Arguing as in the proof of Lemma \ref{lem2.4}, one obtains that $J(u)=c$. On the other hand, by Fatou's lemma we conclude that \begin{align*} c&=J(u)-\frac{1}{4}\langle J'(u),u\rangle \\ &=\frac{1}{4}\|u\|^2+\frac{1}{12}\int_{\mathbb{R}^3}K(x)|u|^6 +\lambda\int_{\mathbb{R}^3}\Big(\frac{1}{4}g(x,u)u-G(x,u)\Big) \\ &\leq \lim_{n\to\infty}\Big[\frac{1}{4}\|u_n\|^2 +\frac{1}{12}\int_{\mathbb{R}^3}K(x)|u_n|^6 +\lambda\int_{\mathbb{R}^3}\Big(\frac{1}{4}g(x,u_n)u_n-G(x,u_n)\Big)\Big]\\ &=\lim_{n\to\infty}\Big(I(u_n)-\frac{1}{4}\langle I'(u_n),u_n\rangle \Big)=c \end{align*} which implies $\|u_n\|\to\|u\|$. Therefore, $u_n\to u$ in $E$. \end{proof} \section{Proof of Theorem \ref{thm13}} In this section, we consider problem \eqref{e1.6} and give the proof of Theorem \ref{thm13}. We shall use the following abstract result which is due to Jeanjean \cite{J}. \begin{lemma} \label{lem4.1} Let $X$ be a Banach space equipped with a norm $\|\cdot\|_X$ and let $\Lambda\in \mathbb{R}^+$ be an interval. Let $\{\Phi_\lambda\}_{\lambda\in \Lambda}$ be a family of $C^1$-functionals on $X$ of the form $$ \Phi_\lambda(u)=A(u)-\lambda B(u),\quad\forall\lambda\in\Lambda, $$ where $B(u)\geq0$ for all $u\in X$ and such that either $A(u)\to+\infty$ or $B(u)\to\infty$, as $\|u\|_X\to\infty$. We assume that there are two points $v_1,v_2$ in $X$ such that $$ c_\lambda :=\inf_{\gamma\in\Gamma}\max_{t\in[0,1]}\Phi_\lambda(\gamma(t)) >\max\{\Phi_\lambda(v_1),\Phi_\lambda(v_2)\},\quad\forall\lambda\in\Lambda $$ where $$ \Gamma=\{\gamma\in C([0,1],X):\gamma(0)=v_1,\gamma(1)=v_2\}. $$ Then, for almost every $\lambda\in\Lambda$, there is a bounded $(PS)_{c_\lambda}$ sequence for $\Phi_{\lambda}$; that is, there exists a sequence $\{u_n(\lambda)\}\subset X$ such that \begin{itemize} \item[(i)] $\{u_n(\lambda)\} $ is bounded in $X$; \item[(ii)] $\Phi_\lambda(u_n(\lambda))\to {c_\lambda}$; \item[(iii)] $\Phi_\lambda'(u_n(\lambda))\to0$ in $X^*$, where $X^*$ is the dual of $X$. Moreover, the map $\lambda\mapsto c_\lambda$ is nonincreasing and left continuous. \end{itemize} \end{lemma} Denote $\Lambda=[\delta,1]$, where $\delta\in(0,1)$ is a positive constant. To apply lemma \ref{lem4.1}, we introduce a family of functions defined by $$ I_\lambda(u)=\frac{1}{2}\int_{\mathbb{R}^3}(a|\nabla u|^2+u^2)+\frac{b}{4}\Big(\int_{\mathbb{R}^3}|\nabla u|^2dx\Big)^2-\lambda\int_{\mathbb{R}^3}F(u) $$ for $\lambda\in[\delta,1]$. First we have the following lemma. \begin{lemma} \label{lem4.2} If {\rm (A2')--(A4')} are satisfied, then \begin{itemize} \item[(i)] there exists a $v\in E\backslash\{0\}$ independent of $\lambda$ such that $I_\lambda(v)\leq0$ for all $\lambda\in [\delta,1]$; \item[(ii)] $c_\lambda=\inf_{\gamma\in\Gamma} \max_{t\in [0,1]}I_\lambda(\gamma(t))>\max\{I_\lambda(0),I_\lambda(v)\}$ for all $\lambda\in [\delta,1]$, where $$ \Gamma=\{\gamma\in C([0,1],E):\gamma(0)=0,\gamma(1)=v\}; $$ \item[(iii)] there exists $M>0$ independent of $\lambda$ such that $c_\lambda\leq M$ for all $\lambda\in[\delta,1]$. \end{itemize} \end{lemma} \begin{proof} (i) For a fixed $u\in E\backslash \{0\}$ and any $\lambda\in [\delta,1]$, we have $$ I_\lambda(u)\leq I_\delta(u)=\frac{1}{2}\int_{\mathbb{R}^3}(a|\nabla u|^2+u^2)+\frac{b}{4}\Big(\int_{\mathbb{R}^3}|\nabla u|^2dx\Big)^2-\delta\int_{\mathbb{R}^3}F(u). $$ Set $u_t(x)=t^2u(\frac{x}{t^2})$, $t>0$. It is easy to check that \begin{equation} \label{e4.1} I_\delta(u_t)=\frac{t^{6}}{2}\int_{\mathbb{R}^3}a|\nabla u|^2+\frac{t^{10}}{2}\int_{\mathbb{R}^3}u^2 +\frac{bt^{12}}{4}\Big(\int_{\mathbb{R}^3}|\nabla u|^2dx\Big)^2-\delta t^{12}\int_{\mathbb{R}^3}\frac{F(t^2u)}{(t^2u)^3}u^3. \end{equation} By (A2') and (A4') if $u\neq0$, then $F(t^2u)/ |t^2u|^3\to+\infty$ as $t\to+\infty$. From Fatou's lemma we have $I_\delta(u_t)\to-\infty$ as $t\to+\infty$. So, taking $v=u_t$, for $t$ large we have $I_\lambda(v)\leq I_\delta(v)<0$ for all $\lambda\in [\delta,1]$. \par (ii) By (A2') and (A3'), for any $\varepsilon>0$ there exists $C_\varepsilon>0$ such that \begin{equation} \label{e4.2} |f(u)|\leq \varepsilon|u|+C_\varepsilon|u|^{p-1},\quad \forall u\in\mathbb{R}. \end{equation} Then, for $\varepsilon$ small enough and by the Sobolev embedding, we obtain \begin{align*} I_\lambda(u) &\geq\frac{1}{2}\int_{\mathbb{R}^3}(a|\nabla u|^2+u^2) -\frac{\varepsilon}{2}\int_{\mathbb{R}^3}u^2 -\frac{C_\varepsilon}{p}\int_{\mathbb{R}^3}u^p\\ &\geq \frac{1-\varepsilon}{2}\|u\|^2-C C_\varepsilon\|u\|^p. \end{align*} Since $p>2$, we deduce that $I_\lambda$ has a strict local minimum at 0 and hence $c_\lambda>0$. (iii) By $c_\lambda\leq\max_{t>0}I_\lambda(u_t)\leq\max_{t>0}I_\delta(u_t)$ for $\lambda\in[\delta,1]$, the conclusion follows from \eqref{e4.1}. \end{proof} Note that conditions (A2')--(A4'), Lemma \ref{lem4.2} and the definition of $I_\lambda(u)$ imply that $I_\lambda(u)$ satisfies the assumptions of Lemma \ref{lem4.1} with $X=E$ and $\Phi_\lambda=I_\lambda$. Hence for almost every $\lambda\in[\delta,1]$, there exists a bounded sequence $u_n(\lambda)\subset E$ such that $$ I_\lambda(u_n(\lambda))\to c_\lambda,\quad I'(u_n(\lambda))\to 0\quad\text{in } E. $$ In the sequel, we denote $\{u_n\}$ in place of $\{u_n(\lambda)\}$ for simplicity. \begin{lemma} \label{lem4.3} Assume $f$ satisfies {\rm(A2')--(A4')}. Let $u$ be a critical point of $I_\lambda$ in $E$, then we have the Pohozaev type identity \begin{equation} \label{e4.3} \frac{a}{2}\int_{\mathbb{R}^3}|\nabla u|^2+\frac{3}{2}\int_{\mathbb{R}^3}u^2+\frac{b}{2} \Big(\int_{\mathbb{R}^3}|\nabla u|^2\Big)^2 -3\lambda\int_{\mathbb{R}^3}F(u)=0. \end{equation} Moreover, there exists $\kappa>0$ independent of $\lambda$ such that $I_\lambda(u)\geq\kappa$ for any nontrivial critical point $u\in E$ of $I_\lambda$. \end{lemma} \begin{proof} The proof of the Pohozaev type identity can be found in \cite{Azzollini1}. Now we show the second conclusion of the lemma. Let $u$ be a nontrivial critical point of $I_\lambda$. Then \begin{equation} \label{e4.4} \|u\|^2+b\Big(\int_{\mathbb{R}^3}|\nabla u|^2\Big)^2 =\lambda\int_{\mathbb{R}^3}f(u)u, \end{equation} which jointly with \eqref{e4.2}, implies that $\|u\|^2\leq \varepsilon\|u\|_2^2+C_\varepsilon\|u\|_p^p$, then for $\varepsilon$ small enough by the Sobolev embedding, one gets that $\|u\|\geq \delta$ for some positive constant $\delta$ independent of $\lambda$. Since $u$ satisfies the Pohozaev type identity \eqref{e4.3} and $\mu>3$ it follows from \eqref{e4.4} and (A4') that \begin{equation} \label{e4.5} \begin{aligned} I_{\lambda}(u) &=\frac{5\mu-6}{12\mu}\int_{\mathbb{R}^3}a|\nabla u|^2 +\frac{\mu-2}{4\mu}\int_{\mathbb{R}^3}u^2 +\frac{(\mu-3)b}{6\mu}\Big(\int_{\mathbb{R}^3}|\nabla u|^2\Big)^2 \\ &\quad +\frac{\lambda}{2}\int_{\mathbb{R}^3}\Big(\frac{1}{\mu}f(u)u-F(u)\Big) \\ &\geq \frac{\mu-2}{4\mu}\|u\|^2\geq\frac{\mu-3}{2\mu-3}\delta^2 :=\kappa>0. \end{aligned} \end{equation} The proof is complete. \end{proof} We need the following global compactness lemma to study the behavior of bounded $(PS)$ sequence of $I_\lambda$, we refer to \cite{LY1} and \cite{LG} for its proof. \begin{lemma} \label{lem4.4} Suppose that (A2')--(A4') hold and let $\{u_n\}\subset E$ be a bounded $(PS)$ sequence of $I_\lambda$ at a certain level $c_{\lambda}>0$. Then, there exists a $u_0\in E$ and $A\in\mathbb{R}$ such that $\widetilde{I}'_{\lambda}(u_0)=0$, where \begin{equation} \label{e4.6} \widetilde{I}_{\lambda}(u)=\frac{a+bA^2}{2}\int_{\mathbb{R}^3}|\nabla u|^2+\frac{1}{2}\int_{\mathbb{R}^3}u^2-\lambda\int_{\mathbb{R}^3}F(u), \end{equation} and either \begin{itemize} \item[(i)] $u_n\to u_0$ in $E$; or \item[(ii)] there exists a positive integer $l\in\mathbb{N}$, and sequence $\{y_n^k\}\subset \mathbb{R}^3$, $k=1,2\ldots l$, with $|y_n^k|\to\infty$, $|y_n^i-y_n^j|\to\infty$, $i\neq j$ as $ n\to\infty$, nonzero critical points $w_1,\ldots,w_l$ of the problem \begin{equation} \label{e4.7} -(a+bA^2)\Delta u+u=\lambda f(u) \end{equation} such that \begin{gather*} c_{\lambda}+\frac{bA^2}{4}=\widetilde{I}_{\lambda}(u_0) +\sum_{k=1}^{l}\widetilde{I}_{\lambda}(w_k), \\ \big\|u_n-u_0-\sum_{k=1}^l w_k(\cdot -y_n^k)\big\|\to 0\quad \text{as } n\to\infty, \end{gather*} where $$ A^2=\|\nabla u_0\|_2^2+\sum_{k=1}^{l}\|\nabla w_k\|_2^2. $$ \end{itemize} \end{lemma} \begin{proposition}\label{prop4.5} Let $\{u_n\}\subset E$ be a bounded $(PS)$ sequence of $I_\lambda$ at a certain level $c_{\lambda}>0$, then exists $u_\lambda\neq0$ such that $I_\lambda'(u_\lambda)=0$. \end{proposition} The proof is similar to \cite[Lemma 3.5]{LY1} and \cite[Lemma 3.4]{LG}, we omit it here. We remark that in this section the nonlinearity $f$ does not satisfy the monotonicity condition (A5), so we can not prove the weak limit of the (PS) sequence of $I_{\lambda}$ is a critical point as we have done in the previous section. Nevertheless, from Lemma \ref{lem4.4} and Proposition \ref{prop4.5}, we can obtain that for almost every $\lambda\in[\delta,1]$, $I_\lambda$ has a nontrivial point $u_\lambda$. Generally speaking, it is not known whether it is true for $\lambda=1$. Motivated by \cite{J}, we can select a sequence $\{\lambda_n\}\in [\delta,1]$ and $u_n\in E\backslash\{0\}$ such that $\lambda_n\to1$ and $I'_{\lambda_n}(u_n)=0$. In order to obtain a nontrivial critical point of $I=I_1$, we need to discuss the critical value $I_{\lambda_n}(u_n)$ carefully. From Lemmas \ref{lem4.1}--\ref{lem4.4}, Proposition \ref{prop4.5}, we have the following result. \begin{lemma} \label{lem4.6} Suppose that (A2')--(A4') hold, then there exists a sequence $\{\lambda_n\}\subset [\delta,1]$ and $u_n\in E\backslash\{0\}$ such that $$ \lambda_n\to1,\;I_{\lambda_n}'(u_n)=0\quad\text{and} \quad \kappa\leq I_{\lambda_n}(u_n)= c_{\lambda_n} $$ Moreover, the sequence $\{u_n\}$ is bounded in $E$. \end{lemma} \begin{proof} We only prove the boundedness of $\{u_n\}$ in $E$, since $I'_{\lambda_n}(u_n)=0$, similar to \eqref{e4.5}, one concludes that \begin{equation} \label{e4.8} \begin{aligned} c_{\lambda_n}&= I_{\lambda_n}(u_n)\\ &=\frac{5\mu-6}{12\mu}\int_{\mathbb{R}^3}a|\nabla u_n|^2+\frac{\mu-2}{4\mu}\int_{\mathbb{R}^3}u_n^2+\frac{(\mu-3)b}{6\mu} \Big(\int_{\mathbb{R}^3}|\nabla u_n|^2\Big)^2 \\ &\quad +\frac{\lambda_n}{2}\int_{\mathbb{R}^3} \Big(\frac{1}{\mu}f(u_n)u_n-F(u_n)\Big) \\ &\geq \frac{\mu-2}{4\mu}\|u_n\|^2. \end{aligned} \end{equation} Recalling that $c_\lambda\leq M$ for all $\lambda\in[\delta,1]$. By Lemma \ref{lem4.2} (iii), from \eqref{e4.8} we see that $\|u_n\|$ is bounded. The proof is complete. \end{proof} \begin{proof}[Proof of Theorem of \ref{thm13}] By Lemma \ref{lem4.6}, we obtain a bounded sequence of nontrivial critical point $\{u_{\lambda_n}\}$ of $I_{\lambda_n}$ such that $\lambda_n\to1$ and $\kappa\leq I_{\lambda_n}(u_{\lambda_n})=c_{\lambda_n}$. Suppose \begin{equation} \label{e4.9} \lim_{n\to\infty}\sup_{y\in\mathbb{R}^3}\int_{B_r(y)}|u_{\lambda_n}|^2=0. \end{equation} Then by Lemma \ref{lem2.3}, $u_{\lambda_n}\to0$ in $L^s(\mathbb{R}^3)$ for all $s\in(2,6 )$, Therefore $$ \int_{\mathbb{R}^3}f(u_{\lambda_n})u_{\lambda_n}\to0 \quad \text{and}\quad \int_{\mathbb{R}^3}F(u_{\lambda_n})\to0. $$ Consequently \begin{equation} \label{e4.10} \begin{aligned} I_{\lambda_n}(u_{\lambda_n}) &=I_{\lambda_n}(u_{\lambda_n})-\frac{1}{2}\langle I'_{\lambda_n}(u_n),u_n\rangle \\ &=-\frac{b}{4}\Big(\int_{\mathbb{R}^3}|\nabla u_{\lambda_n}|^2dx\Big)^2 +\lambda_n\int_{\mathbb{R}^3}\Big(\frac{1}{2}f(u_{\lambda_n})u_{\lambda_n} -F(u_{\lambda_n})\Big)\leq0 \end{aligned} \end{equation} for $n$ large enough. This contradicts to the fact $I_{\lambda_n}(u_{\lambda_n})\geq \kappa$. Hence \eqref{e4.9} does not hold. Then up to a subsequence, we may assume $u_{\lambda_n}\rightharpoonup u_0$ for some $u_0\in E\backslash\{0\}$. By Lemma \ref{lem4.1} (iii), we see that $$ \lim_{n\to\infty}I_1(u_{\lambda_n}) =\lim_{n\to\infty}\Big(I_{\lambda_n}(u_{\lambda_n})+(\lambda_n-1) \int_{\mathbb{R}^3}F(u_{\lambda_n})\Big) =\lim_{n\to\infty}c_{\lambda_n}=c_1 $$ and, for any $\varphi\in H^1(\mathbb{R}^3)$ it follows in a standard way that $$ \lim_{n\to\infty}\langle I'_1(u_{\lambda_n}),\varphi\rangle =\lim_{n\to\infty}\Big(\langle I'_{\lambda_n}(u_{\lambda_n}), \varphi\rangle-(\lambda_n-1)\int_{\mathbb{R}^3}f(u_{\lambda_n})\varphi\Big)=0 $$ which implies $u_{\lambda_n}$ is a bounded $(PS)_{c_1}$ sequence for $I=I_1$. Then by Proposition \ref{prop4.5}, there exists a nontrivial critical point for $I$ and $I(u_0)=c_1$. To prove the existence of ground state solutions, we set $$ m=\inf\{I(u) : u\in E\backslash\{0\}, I'(u)=0\}. $$ It follows from Lemma \ref{lem4.3} that $\kappa\leq m\leq I(u_0)$, where $u_0$ is the nontrivial critical point obtain above. Suppose that $\{u_n\}\in E\backslash\{0\}$ such that $I(u_n)\to m$ and $I'(u_n)=0$. Similar to \eqref{e4.5}, we obtain that $\{u_n\}$ is bounded. Furthermore, as we analyze in \eqref{e4.9}, \eqref{e4.10} the sequence $\{u_n\}$ can not be vanishing. Then up to translation, a subsequence of $\{u_n\}$ still denoted by $\{u_n\}$, converges weakly to $u\in E\backslash\{0\}$. By Lemma \ref{lem4.4} and Proposition \ref{prop4.5} we see that $u$ is a nontrivial critical point of $I$, and $I(u)\geq m$. In order to complete the proof, it suffices to show that $I(u)\leq m$. Indeed, since $I'(u_n)=0, I'(u)=0$, as in \eqref{e4.5}, by Fatou's lemma, we have \begin{align*} m+o(1)&=I(u_n)\\ &=\frac{5\mu-6}{12\mu}\int_{\mathbb{R}^3}a|\nabla u_n|^2 +\frac{\mu-2}{4\mu}\int_{\mathbb{R}^3}u_n^2 +\frac{(\mu-3)b}{6\mu}\Big(\int_{\mathbb{R}^3}|\nabla u_n|^2\Big)^2 \\ &\quad +\frac{1}{2}\int_{\mathbb{R}^3}\Big(\frac{1}{\mu}f(u_n)u_n-F(u_n)\Big)\\ &\geq\frac{5\mu-6}{12\mu}\int_{\mathbb{R}^3}a|\nabla u|^2 +\frac{\mu-2}{4\mu}\int_{\mathbb{R}^3}u^2 +\frac{(\mu-3)b}{6\mu}\Big(\int_{\mathbb{R}^3}|\nabla u|^2\Big)^2 \\ &\quad +\frac{1}{2}\int_{\mathbb{R}^3}\Big(\frac{1}{\mu}f(u)u-F(u)\big) \\ &=I(u)+o(1) \end{align*} which implies $I(u)\leq m$, as required. \end{proof} \subsection*{Acknowledgments} The authors would like to express their sincere gratitude to the referee for careful reading the manuscript and valuable comments and suggestions. The second author would like to thank Prof. G. M. Figueiredo for many valuable discussions on Kirchhoff problems in 2012. This work is supported by the NSFC (11371212, 11171341, 11271386). \begin{thebibliography}{99} \bibitem{AC} C. O. Alves, F. J. S. A. Corr\^{e}a; On existence of solutions for a class of problem involving a nonlinear operator, \emph{Comm. Appl. Nonlinear Anal.}, 8 (2001), 43-56. \bibitem{ACM} C. O. Alves, F. J. S. A. Corr\^{e}a, T. F. Ma; Positive solutions for a quasilinear elliptic equation of Kirchhoff type, \emph{Comput. Math. Appl.}, 49 (2005), 85-93. \bibitem{AF} C. O. Alves, G. M. Figueiredo; Nonlinear perturbations of a periodic Kirchhoff equation in $\mathbb{R}^N$, \emph{Nonlinear Anal.}, 75 (2012), 2750-2759. \bibitem{AP} A. Arosio, S. Panizzi; On the well-posedness of the Kirchhoff string, \emph{Trans. Amer. Math. Soc.}, 348 (1996), 305-330. \bibitem{Azzollini} A. Azzollini; The elliptic Kirchhoff equation in $\mathbb{R}^N$ perturbed by a local nonlinearity, \emph{Differential Integral Equations}, 25 (2012), 543-554. \bibitem{Azzollini1} A. Azzollini; A note on the elliptic Kirchhoff equation in $\mathbb{R}^N$ perturbed by a local nonlinearity, \emph{Commun. Contemp. Math.}, 17 (2015), 1450039, 5 pp. \bibitem{BN} H. Brezis, L. Nirenberg; Positive solutions of nonlinear elliptic equations involving critical Soobolev exponents, \emph{Comm. Pure Appl. Math.}, 36 (1983), 437-477. \bibitem{CDS} M. M. Cavalcanti, V. N. Domingos, J. A. Soriano; Global existence and uniform decay rates for the Kirchhoff-Carrier equation with nonlinear dissipation, \emph{Adv. Differential Equations}, 6 (2001), 701-730. \bibitem{CKW} C. Chen, Y. Kuo, T. Wu; The Nehari manifold for a Kirchhoff type problem involving sign-changing weight functions, \emph{J. Differential Equations}, 250 (2011), 1876-1908. \bibitem{AS} P. D'Ancona, S. Spagnolo; Global solvability for the degenerate Kirchhoff equation with real analytic data, \emph{Invent. Math.}, 108 (1992), 247-262. \bibitem{PF} M. del Pino, P. Felmer; Local Mountain Pass for semilinear elliptic problems in unbounded domains, \emph{Calc. Var. Partial Differential Equations}, 4 (1996), 127-149. \bibitem{FIS} G. Figueiredo, N. Ikoma, J. R. Santos; Existence and concentration result for the Kirchhoff type equations with general nonlinearities, \emph{Arch. Ration. Mech. Anal.}, 213 (2014), 931-979. \bibitem{GL} F. Gazzola, M. Lazzarino; Existence results for general critical growth semilinear elliptic equations, \emph{Commun. Appl. Anal.}, 4 (2000), 39-50. \bibitem{HLP} Y. He, G. Li, S. Peng; Concentrating bound states for Kirchhoff type problems in $\mathbb{R}^3$ involving critical Sobolev exponents, \emph{Adv. Nonlinear Stud.}, 14 (2014), 483-510. \bibitem{HZ1} X. He, W. Zou; Existence and concentration behavior of positive solutions for a Kirchhoff equation in $\mathbb{R}^3$, \emph{J. Differential Equations}, 252 (2012), 1813-1834. \bibitem{HZ2} X. He, W. Zou; Ground states for nonlinear Kirchhoff equations with critical growth, \emph{Ann. Mat. Pura Appl.}, 193 (2014), 473-500. \bibitem{HZ3} X. He, W. Zou; Infinitely many positive solutions for Kirchhoff-type problems, \emph{Nonlinear Anal.}, 70 (2009), 1407-1414. \bibitem{J} L. Jeanjean; On the existence of bounded Palais-Smale sequence and application to a Landesman-Lazer type problem set on $\mathbb{R}^N$, \emph{Proc. Roy. Soc. Edinburgh Sect. A.}, 129 (1999), 787-809. \bibitem{JW} J. Jin, X. Wu; Infinitely many radial solutions for Kirchhoff-type problems in $\mathbb{R}^N$, \emph{J. Math. Anal. Appl.}, 369 (2010), 564-574. \bibitem{K} G. Kirchhoff; \emph{Mechanik}, Teubner, Leipzig, 1883. \bibitem{Lions} J. L. Lions; On some questions in boundary value problems of mathematical physics, in: Contemporary Development in Continuum Mechanics and Partial Differential Equations, in: North-Holl and Math. Stud., vol. 30, North-Holland, Amsterdam, NewYork, 1978, pp. 284-346. \bibitem{LY1} G. Li, H. Ye; Existence of positive ground state solutions for the nonlinear Kirchhoff type equations in $\mathbb{R}^3$, \emph{J. Differential Equations}, 257 (2014), 566-600. \bibitem{LY2} G. Li, H. Ye; Existence of positive solutions for nonlinear Kirchhoff type problems in $\mathbb{R}^3$ with critical Sobolev exponent, \emph{Math. Meth. Appl. Sci.}, (2014), 2570-2584. \bibitem{LG} Z. Liu, S. Guo; Existence of positive ground state solutions for Kirchhoff type problems, \emph{Nonlinear Anal.}, 120 (2015), 1-13. \bibitem{MR} T. Ma, J. Rivera; Positive solutions for a nonlinear nonlocal elliptic transmission problem, \emph{Appl. Math. Lett.}, 16 (2003), 243-248. \bibitem{MZ} A. Mao, Z. Zhang; Sign-changing and multiple solutions of Kirchhoff type problems without P. S. condition, \emph{Nonlinear Anal.}, 70 (2009), 1275-1287. \bibitem{NSY} E. S. Noussair, C. A. Swanson, J. Yang; Positive finite energy solutions of critical semilinear elliptic problems, \emph{Can. J. Math.}, 44 (1992), 1014-1029. \bibitem{PZ} K. Perera, Z. Zhang; Nontrivial solutions of Kirchhoff-type problems via the Yang index, \emph{J. Differential Equations}, 221 (2006), 246-255. \bibitem{R1} P. H. Rabinowitz; On a class of nonlinear Schr\"{o}dinger equations, \emph{Z. Ang. Math. Phys.} 43(1992), 270-291. \bibitem{R2} P. H. Rabinowitz; Minimax Methods in Critical Point Theory with Applications to Differential Equations, CBMS, Reg. Conf. Ser. Math. vol. 65, American Mathematical Society, Providence, RI, 1986. \bibitem{Sq1} M. Squassina; \emph{On the Struwe-Jeanjean-Toland monotonicity trick}, Proc. Roy. Soc. Edinburgh, Sect. A 142 (2012), 155-169. \bibitem{Sq2} M. Squassina; \emph{Symmetry in variational principles and applications}, J. London Math. Soc., 85 (2012), 323-348. \bibitem{SM} J. Sun, S. Ma; Ground state solutions for some Schrodinger-Poisson systems with periodic potentials, \emph{J. Differential Equations}, 260(2016), 2119-2149. \bibitem{SW1} A. Szulkin, T. Weth; Ground state solutions for some indefinite variational problems, \emph{J. Funct. Anal.}, {\bf 257} (2009), 3802-3822. \bibitem{SW2} A. Szulkin, T. Weth; The method of Nehari manifold, Handbook of Nonconvex Analysis and Applications, edited by D. Y. Gao and D. Montreanu. Inernational Press, Boston (2010), 2314-2351. \bibitem{WTXZ} J. Wang, L. Tian, J. Xu, F. Zhang; Multiplicity and concentration of positive solutions for a Kirchhoff type problem, \emph{J. Differential Equations}, 253 (2012), 2314-1351. \bibitem{W} M. Willen; \emph{ Minimax Theorems}, Birkh\"{a}ser, Boston, 1996. \bibitem{Wu} X. Wu; Existence of nontrivial solutions and high energy solutions for Schroinger-Kirchhoff-type equations in $\mathbb{R}^N$, \emph{Nonlinear Anal. Real World Appl.}, 12 (2011), 1278-1287. \bibitem{ZP} Z. Zhang, K. Perera; Sign changing solutions of Kirchhoff type problems via invariant sets of descent flow, \emph{J. Math. Anal. Appl.}, 317 (2006), 456-463. \end{thebibliography} \end{document}