\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2016 (2016), No. 166, pp. 1--11.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu}
\thanks{\copyright 2016 Texas State University.}
\vspace{8mm}}

\begin{document}
\title[\hfilneg EJDE-2016/166\hfil
 Critical quasilinear Schr\"odinger equation]
{Critical quasilinear Schr\"odinger equation with sign-changing potential}

\author[L.-L. Wang, Z.-Q. Han \hfil EJDE-2016/166\hfilneg]
{Li-Li Wang, Zhi-Qing Han}

\address{Li-Li Wang \newline
School of Mathematical Sciences,
Dalian University of Technology,
116024 Dalian,  China. \newline
School of Mathematics,
Tonghua Normal Uninversity,
134002 Tonghua, Jilin,  China}
\email{lili\_wang@aliyun.com}

\address{Zhi-Qing Han (corresponding author)\newline
School of Mathematical Sciences,
Dalian University of Technology,
116024 Dalian,  China}
\email{hanzhiq@dlut.edu.cn}

\thanks{Submitted December 24, 2014. Published June 28, 2016.}
\subjclass[2010]{35A01, 35A15, 35Q55}
\keywords{Quasilinear Schr\"odinger equation; critical growth;
\hfill\break\indent sign-changing potential; mountain pass theorem; $(PS)_c$ sequence}

\begin{abstract}
 We study the existence of nontrivial solutions for a class of
 quasilinear Schr\"odinger equations in $\mathbb {R}^N$ with critical
 nonlinearity, where the potential is allowed to change signs.
 The quasilinear equations are reduced to semilinear equations by using
 a change of variable. The geometric hypotheses of a mountain pass theorem
 without compactness conditions are satisfied so that the equation
 possesses a nontrivial solution.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\allowdisplaybreaks

\section{Introduction}

In this article we discuss the existence of  nontrivial solutions for 
quasilinear Schr\"odinger equation
\begin{equation}
\label{f4}
-\Delta u+V(x)u-\Delta(u^{2})u=f(x,u),\quad x\in \mathbb{R}^N,
\end{equation}
which has atracted a great deal of attention during recent years
(see \cite{cassani2010existence,colin2003stability,colin2004solutions,
deng2011infinitely,fang2014existence,liu2003soliton,
liu2004solutions,liu2003soliton1,moameni2006existence,moameni2007class,
poppenberg2002existence,severo2010solitary,wang2012bound}), because not only it provides an
important model for developing mathematical methods but it represents a
special case of modeling for many physical phenomena, see
\cite{cassani2010existence,poppenberg2002existence} for an explanation.
Some existence results for \eqref{f4} have been concluded when the potential
$V(x)$ is bounded from below or coercive, we refer to
\cite{liu2003soliton,liu2003soliton1,poppenberg2002existence} where they have
focused on the existence of solutions for \eqref{f4} in the subcritical
case when $f(x,u)=|u|^{p-1}u$, $4\le p+1<22^*$, $N\ge3$, and have suggested
the results by using direct variational methods, such as constrained minimization
arguments. To overcome the undefiniteness of natural functional associated
to \eqref{f4}, we rewrite the functional with a new variable which reduces
the problem to looking for solutions of an auxiliary semilinear equation
by employing the ideas in \cite{colin2004solutions,severo2010solitary,liu2003soliton}.
 We establish a new potential function $V(x)$ which can be sign-changing and
may be unbounded from below without any periodic hypotheses.
 A new nonlinearity $f(x,u)=K(x)|u|^{22^*-2}u+g(x,u)+h(x)$ is established
which is more general than in other papers, for example
\cite{colin2003stability,fang2014existence,liu2003soliton,liu2003soliton1,
poppenberg2002existence,tang2013infinitely,zhang2011multiplicity}.

First we consider the following quasilinear Schr\"odinger equation with 
critical growth
\begin{equation} \label{f1}
-\Delta u+V(x)u-\Delta(u^{2})u=K(x)|u|^{22^*-2}u+g(x,u)+h(x),\quad x\in \mathbb{R}^N,
\end{equation}
where the functions $V$, $K$, $h$ $:\mathbb{R}^N\to \mathbb{R}$ and
$g:\mathbb{R}^N\times\mathbb{R}\to\mathbb{R}$ are continuous and
satisfy the following assumptions:
\begin{itemize}

\item[(A1)] $\int |\nabla u|^2+V(x)u^2>0$ for all $u\in E\setminus \{0\}$.

\item[(A2)] $V(x)$ is sign-changing, $V^+(x)\in L^{\infty}(\mathbb{R}^N)$,
 $\lim_{|x|\to+\infty}V^+(x)=a_0>0$ and
 $\|V^{-}\|_{N/2}<\frac{S(\theta -4)}{\theta -2}$, where
 $V^{\pm}(x):=\max\{\pm V(x),0\}$, $S$ denotes the Sobolev optimal
 constant and $\theta$ is the constant in (A6).

\item[(A3)] $0< C\le K(x)\in L^{\infty}(\mathbb{R}^N)$.

\item[(A4)] $g(x,u)=o(u)$ uniformly in $x\in\mathbb{R}^N$ as $u\to 0^+$.

\item[(A5)] There are constants $a_1, a_2>0$ and $4\le p<22^*$ such that
$$
|g(x,u)|\leq a_1+a_2|u|^{p-1},\quad\forall (x,u)\in \mathbb{R}^N\times[0,+\infty).
$$

\item[(A6)] There exists a constant $\theta\in(4,22^*)$ satisfying
$$
0< G(x,u)\leq \frac{1}{\theta} g(x,u)u,\quad
\forall (x,u)\in \mathbb{R}^N\times(0,+\infty),
$$
where $G(x,u):=\int_{0}^{u}g(x,s)ds$.

\item[(A7)] $h\not\equiv 0$ and $\|h\|_{2N/(N+2)}<\frac{\alpha}{4}S^{1/2}\rho $,
 where $\alpha$ and $\rho$ are given in Lemma \ref{lem3.3}.

\end{itemize}
We remark that the potential may be unbounded from below and the associated 
functional does not satisfy any compactness conditions. Note that 
$22^*=\frac{4N}{N-2}$, here and in the sequel, $N\ge3$.
Let
$$
E:=\{u\in H^1(\mathbb{R}^N):\int V^{+}(x)u^{2}<\infty\},
$$
we observe that $E$ is a Hilbert space equipped with the inner product
$$
(u,v):=\int \nabla u \nabla v+V^+(x)uv
$$
and the norm $\|u\|=(u,u)^{1/2}$. Obviously, it follows from (A2) that
$\|\cdot\|$ is an equivalent norm with the standard one in $H^1(\mathbb{R}^N)$ 
and hence $E$ is continuously embedded into $L^p(\mathbb{R}^N)$, 
$2\le p\le 2^*$, i.e., there is a constant $\tau_p>0$ such that
\begin{equation} \label{f2}
\|u\|_p\le\tau_p\|u\|,\quad \forall u\in E,
\end{equation}
where $\|\cdot\|_p$ is used for the usual norm in $L^p(\mathbb{R}^N)$.
Now we state our main result.

\begin{theorem} \label{thm1.1}
If the conditions {\rm (A1)--(A7)} hold. Then problem \eqref{f1} possesses
 a nontrivial nonnegative solution in $E$.
\end{theorem}

Also, we consider a more general problem
\begin{equation} \label{p2}
-\Delta u+V(x)u-\Delta(u^{2})u=|u|^{22^*-2}u+g(u),\quad x\in \mathbb{R}^N,
\end{equation}
under  hypotheses (A1) and
\begin{itemize}
\item[(A8)] $V(x)$ is sign-changing,
 $\lim_{|x|\to+\infty}V^+(x)=V^+(\infty)>0$,
 $V^+(x)\leq V^+(\infty)$ in $\mathbb{R}^N$ and
 $\|V^{-}\|_{N/2}<\frac{S(\theta -4)}{\theta -2}$.

\item[(A4')] $g(u)=o(u)$ as $u\to 0^+$.

\item[(A5')] There are constants $a_1, a_2>0$ and $4\le p<22^*$ such that
$$
|g(u)|\leq a_1+a_2|u|^{p-1},~\forall u\in [0,+\infty).
$$

\item[(A6')] There exists a constant $\theta\in(4,22^*)$ with
$$
0< G(u)\leq \frac{1}{\theta} g(u)u,~\forall u\in (0,+\infty),
$$
where $G(u)=\int_0^ug(s)ds$.

\item[(A9)]
$(i) G(u)/(u^{22^*-1})\to{+\infty}$ as ${u\to+\infty}$,  if $3\le N<10$;\\
$(ii) G(u)/ u^4 \to{+\infty}$ as ${u\to+\infty}$, if $N\ge 10$.

\item[(A10)] The function $\frac{g(u)}{u^3}$ is nondecreasing for all $u>0$.

\end{itemize}
Now we state the second main result.

\begin{theorem} \label{thm1.2} 
Assume that {\rm (A1), (A8), (A4')--(A6'), (A9), (A10)}  are satisfied. 
Then problem \eqref{p2} admits a nontrivial nonnegative solution in $E$.
\end{theorem}

\begin{remark} \label{rmk1} \rm
 Regarding the the results suggested in \cite{fang2014existence},
Theorems \ref{thm1.1} and \ref{thm1.2} give an extension from their
 results to quasilinear Schr\"odinger equation including critical terms case.
\end{remark}

\begin{remark} \label{rmk2} \rm
A problem of type \eqref{p2} for $N=2$ was studied  in \cite{moameni2007class} 
where $V$ and $g$ are two continuous 1-periodic functions, $V$ is nonnegative 
and bounded from below and $g$ is critical growth. 
Moreover in \cite{silva2010quasilinear} a similar result to Theorem 
\ref{thm1.2} is provided under a more restricted hypotheses on
the periodic potential $V$. While our results in both Theorems \ref{thm1.1}
and \ref{thm1.2} do not need any periodic conditions and the potential
$V(x)$ may be unbounded from below. Also, the method of our proof is
 different from that in \cite{silva2010quasilinear}.
\end{remark}

The article is organized as follows: in Section 2,
 we reduce the quasilinear problem into a semilinear one by the dual method 
and show some preliminary results. 
Section 3 is devoted to prove that the mountain pass level of $I$ is well defined, 
show the boundedness for the $(PS)_c$ sequence of the associated functional, 
and finish Theorem \ref{thm1.1}.
Finally we bring results that complete the proof of Theorem \ref{thm1.2}
in Section 4.

Throughout this article, $C$ will denote various positive constants whose
 exact value is not essential. The domain of an integral is $\mathbb{R}^N$ 
unless otherwise indicated. $\int f(x)dx$ is abbreviated to $\int f(x)$.


\section{Preliminary results} \label{pr}

We show that the energy functional corresponding to \eqref{f1} given by
\begin{align*}
J(u) :=&\frac{1}{2}\int (1+2u^{2})|\nabla u|^{2}
 +\frac{1}{2}\int V(x)u^{2}-\frac{1}{22^*}\int K(x)|u|^{22^*} \\
&-\int G(x,u)-\int h(x)u,
\end{align*}
which is not well defined in general, such as in $H^1(\mathbb{R}^N)$. 
To avoid this trouble, we use of the change of variable 
$v:=f^{-1}(u)$ introduced by \cite{liu2003soliton}, where $f$ is defined by
\[
f'(t)=\frac{1}{\sqrt{1+2f^{2}(t)}} \text{ on } [0,+\infty) \text{ and } 
f(t)=-f(-t) \text{ on } (-\infty ,0].
\]
We list some properties of $f$, and the proofs of which may be found in 
\cite{colin2004solutions,severo2010solitary}.

\begin{lemma} \label{lem2.1} The function $f$ satisfies the following properties:
\begin{itemize}
  \item[(1)] $f$ is uniquely defined, $C^{\infty}$ and invertible;
  \item[(2)] $|f'(t)|\leq 1$ for all $t\in \mathbb{R}$; 
  \item[(3)] $|f(t)|\leq |t|$ for all $t\in \mathbb{R}$;
  \item[(4)] $f(t)/t\to 1$ as $t\to 0$;
  \item[(5)] $f(t)/\sqrt{t}\to 2^{1/4}$ as $t\to +\infty$;
  \item[(6)] $f(t)/2\leq tf'(t)\leq f(t)$ for all $t\ge0$;
  \item[(7)] $|f(t)|\leq 2^{1/4}|t|^{1/2}$ for all $t\in \mathbb{R}$;
  \item[(8)] there exists a positive constant $C$ such that 
 $|f(t)|\geq C|t|$ for $|t|\leq 1$ and $|f(t)|\geq C|t|^{1/2}$ for $|t|\geq 1$;
  \item[(9)] $|f(t)f'(t)| < 1/\sqrt{2}$ for all $t\in \mathbb{R}$;
  \item[(10)] the function $f(t)t^{-1}$ is nonincreasing for all 
 $t\in\mathbb{R}\backslash \{0\}$;
  \item[(11)] the function $f(t)f'(t)t^{-1}$ is decreasing for all $t>0$;
  \item[(12)] the function $f^3(t)f'(t)t^{-1}$ is increasing for all $t>0$;
  \item[(13)] the function $f^{22^*-1}(t)f'(t)t^{-1}$ is increasing for all $t>0$.
\end{itemize}
\end{lemma} 

After the change of variable we obtain the  functional
\begin{align*}
I(v):=&\frac{1}{2}\int |\nabla v|^{2}+\frac{1}{2}\int V(x)f^{2}(v)
 -\frac{1}{22^*}\int K(x)|f(v)|^{22^*}\\
&-\int G(x,f(v))-\int h(x)f(v).
\end{align*}
Then $I$ is well-defined on $E$ and belongs to $C^{1}$ in view of the hypotheses
 (A2)--(A5) and (A7).
Furthermore, it is easy to check that
\begin{align*}
\langle I'(v),w\rangle
&=\int  \nabla v \nabla w+\int V(x)f(v)f'(v)w-\int K(x)|f(v)|^{22^*-2}f(v)f'(v)w\\
&\quad -\int g(x,f(v))f'(v)w-\int h(x)f'(v)w,\quad \forall v,w\in E,
\end{align*}
and the critical point of $I$ are weak solutions of the problem
\[
-\Delta v+V(x)f(v)f'(v)=K(x)|f(v)|^{22^*-2}f(v)f'(v)+g(x,f(v))f'(v)+h(x)f'(v),
\]
for $x\in \mathbb{R}^N$.
We observe that if $v\in E$ is a critical point of the functional $I$, 
then the function $u=f(v)\in E$ is a solution of 
\eqref{f1} (cf:\cite{colin2004solutions}). To obtain a nonnegative solution 
for \eqref{f1}, we set $g(x,u)=0$ for all $x\in\mathbb{R}^N$ and $u\le0$. 
By (A4) and (A5) we also see that, given $\varepsilon>0$ there exists a 
constant $C_\varepsilon>0$ such that
\begin{equation} \label{p3}
|g(x,u)|\le \varepsilon|u|+C_\varepsilon|u|^{p-1},\quad
\forall (x,u)\in\mathbb{R}^N\times\mathbb{R}.
\end{equation}


\section{Proof of Theorem \ref{thm1.1}} \label{t1}

In this section we assume that (A1)--(A7) are satisfied. 
The following lemmas are crucial for the proof of Theorem \ref{thm1.1}.

\begin{lemma} \label{lem3.2}
 There exist constants $\rho,\alpha>0$ such that 
$\int |\nabla v|^{2}+ V(x)f^2(v)\geq \alpha \|v\|^2$, whenever $\|v\|=\rho$. 
 \end{lemma}

The proof of the above lemma is similar to that of 
\cite[Lemma 3.1]{fang2014existence}. So we omit it.


\begin{lemma} \label{lem3.3} 
For the above $\rho$, there exists a constant $\beta >0$ such that 
$\inf_{\|v\|=\rho}I(v)\geq \beta $.
 \end{lemma}

\begin{proof}
By (A3), Lemma \ref{lem2.1}(7) and the Sobolev imbedding inequality,
it is easy to obtain
\begin{align*}
\int K(x)|f(v)|^{22^*}&\le 2^{2^*/2}\|K\|_\infty \int |v|^{2^*}\\
&\le 2^{2^*/2}\|K\|_\infty S^{-2^*/2}
\Big(\int |\nabla v|^2\Big)^{2^*/2}\\
&\le 2^{2^*/2}\|K\|_\infty S^{-2^*/2}\| v\|^{2^*}.
\end{align*}
By \eqref{p3}, Lemma \ref{lem2.1}(3,7) and \eqref{f2}, we have
\begin{align*}
\int G(x,f(v))&\le \frac{\varepsilon}{2}\int |f(v)|^2
 +\frac{C_\varepsilon}{p}\int |f(v)|^p\\
&\le \frac{\varepsilon}{2}\int |v|^2+C_\varepsilon\int |v|^{p/2}\\
&\le \frac{\varepsilon}{2}\tau^2_2\|v\|^2+C_\varepsilon\|v\|^{p/2}.
\end{align*}
It follows from (A7), Lemma \ref{lem2.1}(3), the H\"older inequality and 
the Sobolev imbedding inequality that
\[
\int h(x)f(v)\le \|h\|_{\frac{2N}{N+2}}\|v\|_{2^*}
\le \|h\|_{\frac{2N}{N+2}}S^{-1/2}(\int|\nabla v|^2)^{1/2}
\le \|h\|_{\frac{2N}{N+2}}S^{-1/2}\|v\|.
\]
Therefore, combining the above inequalities with Lemma \ref{lem3.2}, we obtain
\[
I(u)\geq \frac{\alpha}{2}\|v\|^2-\frac{2^{2^*/2}}{22^*}
\|K\|_\infty S^{-2^*/2}\|v\|^{2^*}
-\frac{\varepsilon}{2}\tau^2_2\|v\|^2-C_\varepsilon\|v\|^{p/2}
-\|h\|_{\frac{2N}{N+2}}S^{-1/2}\|v\|.
\]
Choosing $\varepsilon\le \alpha/(2\tau^2_2)$ and for every $\|v\|=\rho$ we obtain
\begin{align*}
I(u)\ge\rho \big[\frac{\alpha}{4}\rho-\|h\|_{\frac{2N}{N+2}}S^{-1/2}\big]
-\frac{2^{2^*/2}}{22^*}\|K\|_\infty S^{-2^*/2}\rho^{2^*}-C\rho^{p/2}.
\end{align*}
For $\rho$ sufficiently small, we derive that there exists a constant 
$\beta>0$ such that $\inf_{\|v\|=\rho}I(v)\geq \beta$ by (A7).
\end{proof}

\begin{lemma} \label{lem3.4} 
There exists $v_0\in E$ such that $\|v_0\|>\rho$ and $I(v_0)< 0$.
\end{lemma}

\begin{proof}
Given $\varphi\in C^\infty_0(\mathbb{R}^N,[0,1])$ with $B:=supp \varphi$,
 we derive that $I(t\varphi)\to -\infty$ as $t\to +\infty$, which 
completes the proof if we take $v_0=t\varphi$ with $t$ large enough. 
Note that $0<t\varphi\le t$ in $B$ and then 
\begin{equation} \label{e3.1}
f(t\varphi)\ge f(t)\varphi
\end{equation}
by Lemma \ref{lem2.1}(10).
It follows from (A2), (A3), (A6), (A7), Lemma \ref{lem2.1}(3) and \eqref{e3.1}
 that
\begin{align*}
I(t\varphi)
&\le\frac{t^2}{2}\int_{B}|\nabla \varphi|^2+\frac{1}{2}\int_{B}V^+(x)f^2(t\varphi)-\frac{1}{2}\int_{B}V^-(x)f^2(t\varphi)\\
&\quad -\frac{C}{22^*} \cdot f^{22^*}(t)\int_{B}|\varphi|^{22^*}+t\|h\|_{\frac{2N}{N+2}}\|\varphi\|_{2^*}\\
&\le\frac{t^2}{2}\|\varphi\|^2-\frac{C}{22^*} \cdot f^{22^*}(t)\int_{B}|\varphi|^{22^*}+t\|h\|_{\frac{2N}{N+2}}\|\varphi\|_{2^*}\\
&\to-\infty\quad \text{as }t\to +\infty,
\end{align*}
since $f^{22^*}(t)/t^2 \to +\infty$ as $t\to +\infty$.
\end{proof}

\begin{lemma} \label{lem3.5} 
The $(PS)_c$ sequence $(v_n)\subset E$ is bounded.
\end{lemma}

\begin{proof}
Set $(v_n)\subset E$ be a $(PS)_c$ sequence:
$I(v_n)\to c$  and $I'(v_n)\to 0$ as $n\to\infty$.
Using Lemma \ref{lem2.1}(3,6), (A3), (A6) and the Sobolev 
imbedding inequality we easily deduce that
\begin{align*}
&c+o_n(1)+o_n(1)\|v_n\|\\
&= I(v_n)-\frac{2}{\theta}I'(v_n)v_n\\
&\ge (\frac{1}{2}-\frac{2}{\theta})\int|\nabla v_n|^2
 + V^+(x)f^2(v_n)-(\frac{1}{2}-\frac{1}{\theta})\int V^-(x)f^2(v_n)\\
&\quad -(\frac{1}{22^*}-\frac{1}{\theta})\int K(x)|f(v_n)|^{22^*}
 +\frac{1}{\theta}\int g(x,f(v_n))f(v_n)\\
&\quad -\int G(x,f(v_n))-(1+\frac{2}{\theta})\int |h(x)f(v_n)|\\
&\ge (\frac{1}{2}-\frac{2}{\theta})\int|\nabla v_n|^2
 + V^+(x)f^2(v_n)-(\frac{1}{2}-\frac{1}{\theta})\|V^-\|_{N/2}\|v_n\|^2_{2^*}\\
&\quad +(\frac{1}{\theta}-\frac{1}{22^*})\int K(x)|f(v_n)|^{22^*}
 -(1+\frac{2}{\theta})\|h\|_{\frac{2N}{N+2}}\|v_n\|_{2^*}\\
&\ge \Big[(\frac{1}{2}-\frac{2}{\theta})
 -(\frac{1}{2}-\frac{1}{\theta})\|V^-\|_{N/2}S^{-1}\Big]
\int|\nabla v_n|^2+ V^+(x)f^2(v_n)\\
&\quad +(\frac{1}{\theta}-\frac{1}{22^*})\int K(x)|f(v_n)|^{22^*}
 -(1+\frac{2}{\theta})\|h\|_{\frac{2N}{N+2}}S^{-1/2}
\Big(\int |\nabla v_n|^2\Big)^{1/2}.
\end{align*}
It follows from (A2) that 
$(\frac{1}{2}-\frac{2}{\theta})-(\frac{1}{2}-\frac{1}{\theta})\|V^{-}
\|_{N/2}S^{-1}>0$ and hence
\begin{equation} \label{32}
\begin{gathered}
\int|\nabla v_n|^2+V^+(x)f^2(v_n)\le C+C\|v_n\|,\\
\int K(x)|f(v_n)|^{22^*}\le C+C\|v_n\|.
\end{gathered}
\end{equation}
From \eqref{32}, we only prove that $\int V^+(x)v_n^2\le C+C\|v_n\|$.
In fact, from (A2), (A3), Lemma \ref{lem2.1}(8) and \eqref{32} it follows that
\begin{align*}
\int_{|v_n|\ge 1}V^+(x)v_n^2
&\le\|V^+\|_\infty\int_{|v_n|\ge 1} v_n^2\le C\|V^+\|_\infty\int |f(v_n)|^{22^*}\\
&\le C\|V^+\|_\infty \int K(x)|f(v_n)|^{22^*}\le C+C\|v_n\|
\end{align*}
and
\[
\int_{|v_n|\le 1}V^+(x)v_n^2\le C\int_{|v_n|\le 1}V^+(x)f^2(v_n)\le C+C\|v_n\|.
\]
Thus we have $\|v_n\|^2\le C+C\|v_n\|$ and then $(v_n)\subset E$ is bounded.
\end{proof}

\begin{lemma} \label{lem3.6}
 Suppose that $(v_n)\subset E$ is a bounded $(PS)_c$ sequence for the functional $I$,
 then up to a subsequence, $v_n\rightharpoonup v$ in $E$ and $v$ is a nontrivial 
critical point of the functional $I$.
\end{lemma}

\begin{proof}
The argument is similar as in \cite{silva2010quasilinear}. 
Since $C^\infty_0(\mathbb{R}^N)$ is dense in $H^1(\mathbb{R}^N)$, 
we only need to show that $\langle I'(v),\varphi\rangle=0$ for all 
$\varphi\in C^\infty_0(\mathbb{R}^N)$. Notice that
 $\langle I'(v_n),\varphi\rangle\to 0$, for all 
$\varphi\in C^\infty_0(\mathbb{R}^N)$, it suffices to derive that 
$\langle I'(v_n),\varphi\rangle\to\langle I'(v),\varphi\rangle$. In fact,
\begin{align*}
&\langle I'(v_n),\varphi\rangle-\langle I'(v),\varphi\rangle
 -\int (\nabla v_n-\nabla v)\nabla \varphi\\
&=\int [f(v_n)f'(v_n)-f(v)f'(v)]V^+(x)\varphi
 +\int[f(v)f'(v)-f(v_n)f'(v_n)]V^-(x)\varphi\\
&\quad+\int\big[|f(v)|^{22^*-2}f(v)f'(v)-|f(v_n)|^{22^*-2}f(v_n)f'(v_n)\big]
 K(x)\varphi\\
&\quad+\int \big[g(x,f(v))f'(v)-g(x,f(v_n))f'(v_n)\big] \varphi
 +\int[f'(v)-f'(v_n)]h(x)\varphi.
\end{align*}
Since $E$ is continuously embedded into $H^1(\mathbb{R}^N)$, we know that
$$
\int \nabla v_n\nabla\varphi\to\int\nabla v\nabla\varphi.
$$
Besides, it follows from $v_n\rightharpoonup v$ in $E$ that $v_n\to v$ 
in $L^p_{\rm loc}(\mathbb{R}^N)$, $p\in [1,2^*)$. 
Then, up to subsequence, $v_n\to v$ a.e. on $B:=supp\varphi$ as $n\to\infty$ 
and $|v_n(x)|\le|w_p(x)|$ a.e. on $B$ with $w_p\in L^p(B)$ for every 
$n\in \mathbb N$. Therefore, we have
\begin{gather*}
f'(v_n)\to f'(v)\quad \text{a.e. on $B$ as $n\to\infty$}, \\
f(v_n)f'(v_n)\to f(v)f'(v)\quad \text{a.e. on  $B$ as $n\to\infty$}, \\
|f(v_n)|^{22^*-2}f(v_n)f'(v_n)\to|f(v)|^{22^*-2}f(v)f'(v)
\quad\text{a.e. on $B$ as $n\to\infty$}, \\
g(x,f(v_n))f'(v_n)\to g(x,f(v))f'(v)\quad
\text{a.e. on $B$ as $n\to\infty$}.
\end{gather*}
Furthermore, by (A2), (A3), (A7), Lemma \ref{lem2.1}(2,7,9) 
 and the H\"older inequality we have
\begin{gather*}
|V^+(x)f(v_n)f'(v_n)\varphi|\le C\|V^+\|_\infty |\varphi|\in L^1(B), \\
|V^-(x)f(v_n)f'(v_n)\varphi|\le|V^-(x)||\varphi|\in L^1(B), \\
|K(x)|f(v_n)|^{22^*-2}f(v_n)f'(v_n)\varphi|
 \le\|K\|_\infty 2^{\frac{2^*-1}{2}}|w_{2^*-1}|^{2^*-1}|\varphi|\in L^1(B), \\
|h(x)f'(v_n)\varphi|\le|h(x)||\varphi|\in L^1(B).
\end{gather*}
Hence, the Lebesgue Dominated Convergence Theorem implies 
\begin{gather*}
\int V^+(x)f(v_n)f'(v_n)\varphi\to \int V^+(x)f(v)f'(v)\varphi, \\
\int V^-(x)f(v_n)f'(v_n)\varphi\to \int V^-(x)f(v)f'(v)\varphi, \\
\int K(x)|f(v_n)|^{22^*-2}f(v_n)f'(v_n)\varphi\to
 \int K(x)|f(v)|^{22^*-2}f(v)f'(v)\varphi, \\
\int h(x)f'(v_n)\varphi\to\int h(x)f'(v)\varphi.
\end{gather*}
For $|v_n|\le 1$, by \eqref{p3} and Lemma \ref{lem2.1}(2,3), we have
\[
|g(x,f(v_n))f'(v_n)\varphi|\le\varepsilon|f(v_n)||\varphi|
+C_\varepsilon|f(v_n)|^{p-1}|\varphi|\le(\varepsilon+C_\varepsilon)|\varphi|.
\]
For $|v_n|> 1$, by \eqref{p3} and Lemma \ref{lem2.1}(2,3,7,9)
 we conclude that
\begin{align*}
|g(x,f(v_n))f'(v_n)\varphi|
&\le\varepsilon|v_n||\varphi|+C_\varepsilon|f(v_n)|^{p-1}|f'(v_n)||\varphi|\\
&\le\varepsilon|w_2||\varphi|+C_\varepsilon|f(v_n)|^{p-2}|\varphi|\\
&\le\varepsilon|w_2||\varphi|+C_\varepsilon|v_n|^{\frac{p}{2}-1}|\varphi|\\
&\le\varepsilon|w_2||\varphi|+C_\varepsilon|w_{2^*-1}|^{2^*-1}|\varphi|.
\end{align*}
Combining the above facts and using the Lebesgue Dominated Convergence Theorem 
implies
$$
\int g(x,f(v_n))f'(v_n)\varphi\to\int g(x,f(v))f'(v)\varphi.
$$
Hence, $v$ is a critical point of $I$. From the condition (A7), $v$ is nontrivial.
\end{proof}


\begin{proof}[Proof of Theorem \ref{thm1.1}]
 Lemmas \ref{lem3.3} and \ref{lem3.4} imply that the functional $I$ 
satisfies the mountain pass geometry, thus the $(PS)_c$ sequence exists, where
$$
c:=\inf_{\gamma\in \Gamma}\max_{t\in [0,1]}I(\gamma(t)),\quad
\Gamma:=\{\gamma\in C([0,1],E):\gamma(0)=0,\gamma(1)=v_0\}.
$$
Assume that $(v_n)\subset E$ is a $(PS)_c$ sequence, $(v_n)$ is bounded 
by Lemma \ref{lem3.4}. Going if necessary to a subsequence, 
$v_n\rightharpoonup v$ in $E$. We obviously get that $v$ is a nontrivial 
critical point of the functional $I$ by Lemma \ref{lem3.6}.
\end{proof}

\section{Proof of Theorem \ref{thm1.2}} \label{s4}

In this section we assume that (A1), (A8), (A4')--(A6'), (A9), (A10)
 are satisfied. We study the existence of nontrivial critical points for 
the functional $I_0\in C^1(E, \mathbb{R})$ given by
$$
I_0(v):=\frac{1}{2}\int|\nabla v|^{2}+\frac{1}{2}\int V(x)f^{2}(v)
-\frac{1}{22^*}\int |f(v)|^{22^*}-\int G(f(v)).
$$
We also denote the corresponding limiting functional by
$$
I_1(v):=\frac{1}{2}\int|\nabla v|^{2}+\frac{1}{2}\int V^+(\infty)f^{2}(v)
-\frac{1}{22^*}\int |f(v)|^{22^*}-\int G(f(v)).
$$
We set $g(u)=0$ if $u\le0$. Some propositions and lemmas are needed and 
their proofs are similar as in \cite{silva2010quasilinear}, we just state
 them in brief and omit their proofs as follows.

\begin{proposition} \label{prop4.1} 
Assume {\rm (A8), (A4'), (A5')} hold. 
Let $(v_n)\subset E$ be a $(PS)_c$ sequence with $0<c<\frac{1}{2N}S^{\frac{N}{2}}$,
 and $v_n\rightharpoonup 0$ in $E$. 
Then there exist a sequence $(y_n)\subset\mathbb{R}^N$ and $r,\eta>0$ 
such that $|y_n|\to+\infty$ and
$$
\limsup_{n\to\infty}\int_{B_r(y_n)}v^2_n\geq\eta>0.
$$
\end{proposition}

Given $\varepsilon>0$, we study the function $w_\varepsilon:\mathbb{R}^N\to\mathbb{R}$ defined by
$$
w_\varepsilon(x)=C(N)\frac{\varepsilon^\frac{N-2}{2}}{(\varepsilon^2+|x|^2)^\frac{N-2}{2}},
$$
where $C(N)=[N(N-2)]^\frac{N-2}{4}$. Recall that by
(\cite{willem1997minimax,ambrosetti2007nonlinear,rabinowitz1986minimax}),
 $\{w_\varepsilon\}_{\varepsilon>0}$ is a family of functions on which the infimum, that 
defines the best constant $S$, for the Sobolev imbedding 
$D^{1,2}(\mathbb{R}^N)\subset L^{2^*}(\mathbb{R}^N)$, is attained. 
Moreover, one has
$$
w_\varepsilon\in L^{2^*}(\mathbb{R}^N),\quad \nabla w_\varepsilon\in L^2(\mathbb{R}^N),\quad
\int|\nabla w_\varepsilon|^{2}=\int|w_\varepsilon|^{2^*}=S^{\frac{N}{2}}.
$$
We also consider $\phi\in C^\infty_0(\mathbb{R}^N, [0,1])$, $\phi\equiv1$ in 
$B_1(0)$, $\phi\equiv0$ in $\mathbb{R}^N\backslash B_2(0)$ and define
$$
u_\varepsilon=\phi w_\varepsilon,\quad v_\varepsilon=\frac{u_\varepsilon}{(\int u_\varepsilon^{2^*})^{1/2^*}}.
$$

\begin{lemma} \label{lem4.1} 
There exist positive constants $k_1$, $k_2$ and $\varepsilon_0$ such that
\begin{gather*}
\int_{\mathbb{R}^N\backslash B_1(0)}|\nabla u_\varepsilon|^2=O(\varepsilon^{N-2})\quad \text{as }\varepsilon\to 0^+,\\
k_1<\int u_\varepsilon^{2^*}<k_2,\quad \forall 0<\varepsilon<\varepsilon_0,\\
\int_{|x|\le1}|x|^{N-2}w_\varepsilon^{2^*}=O(\varepsilon^{N-2})\quad \text{as }\varepsilon\to 0^+,\\
\int|\nabla v_\varepsilon|^2\le S+O(\varepsilon^{N-2})\quad \text{as } \varepsilon\to 0^+.
\end{gather*}
\end{lemma}

\begin{lemma} \label{lem4.2} As $\varepsilon\to 0$, we have
\begin{gather*}
\|v_\varepsilon\|^2_2=
\begin{cases}
O(\varepsilon), & \text{if } N=3,\\
O(\varepsilon^2|\log\varepsilon|),&\text{if } N=4,\\
O(\varepsilon^2), &\text{if } N\ge5,
\end{cases}
\\
\|v_\varepsilon\|^{2^*-\frac{1}{2}}_{2^*-\frac{1}{2}}=O(\varepsilon^{\frac{N-2}{4}}).
\end{gather*}
\end{lemma}

\begin{proposition} \label{prop4.2} 
If  conditions {\rm  (A4'), (A5'), (A8), (A9)} hold. 
Then there exists $v\in E\backslash \{0\}$ such that
$$
\max_{t\ge 0}I_0(tv)<\frac{1}{2N}S^{\frac{N}{2}}.
$$
\end{proposition}

\begin{lemma} \label{lem4.3} 
If $\{v_n\}\subset E$ is a bounded $(PS)_c$ sequence for the functional $I_0$,
 then up to a subsequence, $v_n\rightharpoonup v\not\equiv 0$ with $I'_0(v)=0$.
\end{lemma}

\begin{proof}
Since $\{v_n\}$ is bounded, going if necessary to a subsequence, 
$v_n\rightharpoonup v$ in $E$. It is obvious that $I'_0(v)=0$. 
If $v\not\equiv 0$, the proof is complete.

If $v=0$, we claim that $\{v_n\}$ is also a $(PS)_c$ sequence for $I_1$. 
Indeed, we have
$$
I_1(v_n)-I_0(v_n)=\frac{1}{2}\int [V^+(\infty)-V^+(x)]f^2(v_n)
+\frac{1}{2}\int V^-(x)f^2(v_n)\to 0,
$$
using (A8), Lemma \ref{lem2.1}(3) and $v^2_n\rightharpoonup 0$ in 
$L^{N/(N-2)}$. Similarly we derive
\begin{align*}
\sup_{\|u\|\leq 1}|\langle I'_1(v_n)-I'_0(v_n),u\rangle|
&=\sup_{\|u\|\leq 1}\big|\int (V^+(\infty)-V^+(x))f(v_n)f'(v_n)u\big|\\
& \quad+\sup_{\|u\|\leq 1}\big|\int V^-(x)f(v_n)f'(v_n)u\big|\to 0.
\end{align*}
In view of Proposition \ref{prop4.2}, we observe that 
$0<\beta_0\le c<\frac{1}{2N}S^{\frac{N}{2}}$, where the constant 
$\beta_0$ will be stated in the proof of Theorem \ref{thm1.2}. 
Furthermore, by Proposition \ref{prop4.1}, there exists a sequence 
$(y_n)\subset \mathbb{R}^N$ and $r,\eta>0$ such that $|y_n|\to+\infty$ and
$$
\limsup_{n\to\infty}\int_{B_r(y_n)}v^2_n\geq\eta>0,~\forall n\in\mathbb N.
$$
Defining $u_n(x)=v_n(x+y_n)$, we know $\{u_n(x)\}$ is also a $(PS)_c$ 
sequence for $I_1$. Thus, going to a subsequence if necessary, 
there exists $u\in E$ such that $u_n\rightharpoonup u$ in $E$ and
 $I'_1(u)=0$ with $u\not\equiv 0$. We obtain that by Fatou's Lemma
\begin{align*}
c=\limsup_{n\to\infty}[I_1(u_n)-\frac{1}{2}I'_1(u_n)u_n]
\geq I_1(u)-\frac{1}{2}I'_1(u)u=I_1(u).
\end{align*}

Our next task is to verify that $\max_{t\ge0}I_1(tu)=I_1(u)\le c$. 
For that, we define the function $\eta(t):=I_1(tu)$ for $t\ge 0$. 
Since $u$ is a critical point of $I_1$, it follows that $u>0$ 
(see the proof in \cite{silva2010quasilinear}). Then we obtain
\begin{align*}
\eta'(t)
&=t\int|\nabla u|^2+\int V^+(\infty)f(tu)f'(tu)u\\
&\quad -\int |f(tu)|^{22^*-2}f(tu)f'(tu)u-\int g(f(tu))f'(tu)u\\
&=t\Big\{\int|\nabla u|^2-\int\Big[\frac{|f(t|u|)|^{22^*-2}f(t|u|)f'(t|u|)}{t|u|}\\
&\quad +\frac{g(f(t|u|))f'(t|u|)}{t|u|}-\frac{V^+(\infty)f(t|u|)f'(t|u|)}{t|u|}
\Big]u^2\Big\}.
\end{align*}
Note that, fixed $x\in\mathbb{R}^N$, the function 
$ \vartheta:(0,+\infty)\to \mathbb{R}$ defined by
\begin{align*}
\vartheta(s)
&=\frac{f^{22^*-1}(s)f'(s)}{s}+\frac{g(f(s))f'(s)}{s}
 -\frac{V^+(\infty)f(s)f'(s)}{s}\\
&=\frac{f^{22^*-1}(s)f'(s)}{s}+\frac{g(f(s))}{f^3(s)}\cdot 
 \frac{f^3(s)f'(s)}{s}+V^+(\infty)(-\frac{f(s)f'(s)}{s})
\end{align*}
is increasing by Lemma \ref{lem2.1}(11,12,13) and (A10). 
Now we observe that $\eta'(1)=0$, since $u$ is a critical point of $I_1$. 
Moreover, we have that $\eta'(t)>0$ for $0<t<1$ and $\eta'(t)<0$ for $t>1$. 
Therefore, $I_1(u)=\eta(1)=\max_{t\ge 0}\eta(t)=\max_{t\ge0}I_1(tu)$ and then
$$
c\le\max_{t\ge0}I_0(tu)\le\max_{t\ge0}I_1(tu)=I_1(u)\le c.
$$
This implies that there exists a way $r_0\in\Gamma$ such that
 $c=\max_{t\in[0,1]}I_0(r_0(t))>0$, and hence, $I_0$ possesses a critical 
point $v$ on level $c$. It follows from $c\ge\beta_0>0=I_0(0)$ that $v$ 
is a nonzero critical point of $I_0$.
\end{proof}

\begin{proof}[Proof of Theorem \ref{thm1.2}]
 The proof is similar as the one of Theorem \ref{thm1.1}. Only we modify the proof 
of Lemma \ref{lem3.3} that
\[
I_0(v)\geq \frac{\alpha}{2}\rho^2-\frac{\varepsilon}{2}
\tau^2_2\rho^2-\frac{2^{2^*/2}}{22^*} S^{-2^*/2}\rho^{2^*}-C_\varepsilon\rho^{p/2},
\]
for every $\|v\|=\rho$. Choosing for all $\varepsilon\in(0,\frac{\alpha}{\tau^2_2})$ 
and $\rho$ sufficiently small, we derive that there exists a constant $\beta_0$ 
such that $\inf_{\|v\|=\rho}I_0(v)\geq \beta_0>0$. 
Combining this fact with Lemma \ref{lem3.4}, the functional $I_0$ has a mountain 
pass geometry. So the $(PS)_c$ sequence $(v_n)$ exists, where
$$
c:=\inf_{r\in \Gamma}\max_{t\in [0,1]}I_0(r(t)),\quad
\Gamma:=\{r\in C([0,1],E):r(0)=0,I_0(r(1))<0\}.
$$
It follows from Lemma \ref{lem3.5} that $(v_n)$ is a bounded $(PS)_c$ 
sequence for the functional $I_0$. Lemma \ref{lem4.3} ensures that 
$I'_0(v)=0$ and $v\not\equiv0$.
\end{proof}

\subsection*{Acknowledgments}
This research was supported by the NSFC 11171047.

\begin{thebibliography}{10}

\bibitem{ambrosetti2007nonlinear}
Antonio Ambrosetti and Andrea Malchiodi; 
\emph{Nonlinear analysis and   semilinear elliptic problems},
 vol. 104, Cambridge University Press, 2007.

\bibitem{cassani2010existence}
Daniele Cassani, Abbas Moameni, et al.; 
\emph{Existence and concentration of   solitary waves for a class of 
quasilinear schr{\"o}dinger equations}, Commun.
  Pure Appl. Anal. \textbf{9} (2010), 281--306.

\bibitem{colin2003stability}
Mathieu Colin et al.;
\emph{Stability of stationary waves for a quasilinear
  schr{\"o}dinger equation in space dimension 2}, Advances in Differential
  Equations \textbf{8} (2003), no.~1, 1--28.

\bibitem{colin2004solutions} Mathieu Colin, Louis Jeanjean;
 \emph{Solutions for a quasilinear
  schr{\"o}dinger equation: a dual approach}, Nonlinear Analysis: Theory,
  Methods \& Applications \textbf{56} (2004), no.~2, 213--226.

\bibitem{deng2011infinitely} Y.B. Deng, S. J. Peng, J. X. Wang;
 \emph{Infinitely many sign-changing solutions
  for quasilinear schr{\"o}dinger equations in rn}, Commun. Math. Sci
  \textbf{9} (2011), no.~3, 859--878.

\bibitem{fang2014existence} Xiang-Dong Fang, Zhi-Qing Han;
 \emph{Existence of nontrivial solutions for a
  quasilinear schrodinger equations with sign-changing potential}, Electronic
  Journal of Differential Equations \textbf{2014} (2014), no.~05, 1--8.

\bibitem{liu2003soliton} Jia-quan Liu, Ya-qi Wang, Zhi-Qiang Wang;
 \emph{Soliton solutions for   quasilinear schr{\"o}dinger equations, ii}, 
Journal of Differential Equations   \textbf{187} (2003), no.~2, 473--493.

\bibitem{liu2004solutions} Jia-quan Liu, Ya-qi Wang, Zhi-Qiang Wang;
\emph{Solutions for quasilinear schr{\"o}dinger equations via the
  nehari method}, Communications in partial differential equations \textbf{29}
  (2004), no.~5-6, 879--901.

\bibitem{liu2003soliton1} Jiaquan Liu, Zhi-Qiang Wang;
\emph{Soliton solutions for quasilinear  schr{\"o}dinger equations, I}, 
Proceedings of the American Mathematical   Society (2003), 441--448.

\bibitem{moameni2006existence} Abbas Moameni;
 \emph{Existence of soliton solutions for a quasilinear
  schr{\"o}dinger equation involving critical exponent in rn}, Journal of
  Differential Equations \textbf{229} (2006), no. 2, 570--587.

\bibitem{moameni2007class} Abbas Moameni;
 \emph{On a class of periodic quasilinear schr{\"o}dinger equations
  involving critical growth in r2}, Journal of mathematical analysis and
  applications \textbf{334} (2007), no.~2, 775--786.

\bibitem{poppenberg2002existence}
Markus Poppenberg, Klaus Schmitt, Zhi-Qiang Wang;
\emph{On the existence of soliton solutions to quasilinear 
schr{\"o}dinger equations}, Calculus of   Variations and Partial Differential 
Equations \textbf{14} (2002), no. 3,   329--344.

\bibitem{rabinowitz1986minimax} Paul H Rabinowitz et al.;
 \emph{Minimax methods in critical point theory with
  applications to differential equations}, no. 65, American Mathematical Soc.,
  1986.

\bibitem{severo2010solitary} Uberlandio Severo et al.;
 \emph{Solitary waves for a class of quasilinear
  schr{\"o}dinger equations in dimension two}, Calculus of Variations and
  Partial Differential Equations \textbf{38} (2010), no.~3-4, 275--315.

\bibitem{silva2010quasilinear} Elves A. B. Silva, Gilberto F. Vieira;
\emph{Quasilinear asymptotically periodic
  schr{\"o}dinger equations with critical growth}, Calculus of Variations and
  Partial Differential Equations \textbf{39} (2010), no.~1-2, 1--33.

\bibitem{tang2013infinitely} X. H. Tang;
 \emph{Infinitely many solutions for semilinear schr{\"o}dinger
  equations with sign-changing potential and nonlinearity}, Journal of
  Mathematical Analysis and Applications \textbf{401} (2013), no.~1, 407--415.

\bibitem{wang2012bound} Youjun Wang, Wenming Zou;
 \emph{Bound states to critical quasilinear
  schr{\"o}dinger equations}, NoDEA: Nonlinear Differential Equations and
  Applications \textbf{19} (2012), no.~1, 19--47.

\bibitem{willem1997minimax} Michel Willem;
\emph{Minimax theorems}, vol. 24, Springer Science \& Business
  Media, 1997.

\bibitem{zhang2011multiplicity} Qingye Zhang, Bin Xu;
\emph{Multiplicity of solutions for a class of   semilinear schr{\"o}dinger 
equations with sign-changing potential}, Journal   of Mathematical Analysis 
and Applications \textbf{377} (2011), no. 2,   834--840.

\end{thebibliography}


\end{document}