\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2018 (2018), No. 11, pp. 1--12.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu}
\thanks{\copyright 2018 Texas State University.}
\vspace{8mm}}

\begin{document}
\title[\hfilneg EJDE-2018/11\hfil Singular quasilinear elliptic equations]
{Solutions to singular quasilinear elliptic equations on bounded domains}

\author[Z. Li, Y. Wang \hfil EJDE-2018/11\hfilneg]
{Zhouxin Li, Youjun Wang}

\address{Zhouxin Li \newline
Department of Mathematics and Statistics,
Central South University,
 Changsha 410083,  China}
\email{lzx@math.pku.edu.cn}

\address{Youjun Wang \newline
Department of Mathematics,
South China University,
Guangzhou 510640, China}
\email{scyjwang@scut.edu.cn}

\dedicatory{Communicated by Claudianor O. Alves}

\thanks{Submitted August 25, 2017. Published January 8, 2018.}
\subjclass[2010]{35J60, 35J65}
\keywords{Quasilinear elliptic equations; critical growth; positive solutions}

\begin{abstract}
 In this article we study  quasilinear elliptic equations
 with a singular operator and at critical Sobolev growth.
 We prove the existence of positive solutions.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\allowdisplaybreaks


\section{Introduction and statement of main results} \label{sec:6-1}


In this article, we study the existence of solutions for the 
 quasilinear elliptic equation
\begin{equation}\label{eq.sch6-main}
\begin{gathered}
    -\Delta u-\kappa\alpha(\Delta(|u|^{2\alpha}))|u|^{2\alpha-2}u
    =|u|^{q-2}u+|u|^{2^*-2}u, \quad \text{in }\Omega, \\
    u>0, \quad \text{in }\Omega, \\
    u=0, \quad  \text{on } \partial\Omega,
  \end{gathered}
\end{equation}
where $\Omega\subset{\mathbb{R}}^N~(N\geq3)$ is an open bounded domain 
with smooth boundary $\partial\Omega$, $0<\alpha<1/2,~2\leq q<2^*$, 
$2^*=\frac{2N}{N-2}$ is the critical Sobolev exponent.

Equation \eqref{eq.sch6-main} comes from mathematical physics and was used 
to model some physical phenomena.
Let us consider the following quasilinear Schr\"{o}dinger equation 
introduced in \cite{LiWa03,LiWW03}
\begin{equation}\label{eq.sch5-1}
  i\partial_tz=-\Delta z+w(x)z-l(|z|^2)z-\kappa\Delta h(|z|^2)h'(|z|^2)z,\quad 
x\in{\mathbb{R}}^N,
\end{equation}
where $w(x)$ is a given potential, $\kappa>0$ is a constant, $N\geq3$.
$h,l$ are real functions of essentially pure power form.

Note that if $\kappa=0$, then \eqref{eq.sch5-1} is the standard semilinear 
Schr\"{o}dinger equation which has been extensively studied,
see \cite{A-M-S, A-D-S} for examples.
For $\kappa>0$, it is a quasilinear problem which has many applications in physics.
The case of $h(s)=s$ was used for the superfluid film equation in plasma physics 
by Kurihura in \cite{Kuri81}.
It also appears in plasma physics and fluid mechanics \cite{LiSe78}, 
in the theory of Heisenberg ferromagnetism and magnons \cite{KoIK90,QuCa82} 
in dissipative quantum mechanics \cite{Hass80} and in condensed matter 
theory \cite{MaFe84}.
The case of $h(s)=s^\alpha,\alpha>0$ was used to models the self-channeling 
of high-power ultrashort laser in matter \cite{BoGa93}.


The study of standing waves to \eqref{eq.sch5-1} of the form
$z(x,t)=\exp(-iet)u(x)$ can reduce to find solutions $u(x)$ to the equation
\begin{equation}\label{eq.sch5-2}
-\Delta u+c(x)u-\kappa\alpha(\Delta(h(|u|^2)))h'(|u|^{2})u
    =l(|u|^2)u,\quad x\in{\mathbb{R}}^N,
\end{equation}
where $c(x)=w(x)-e$ is a new potential function.

In recent years, problems with $h(s)=s$ have been extensively studied under 
different conditions imposed on the potential $c(x)$ and the  perturbation $l(u)$, 
one can refer to \cite{CoJe04,OMS07,OMS10,LiWW03} and some references therein.
Note that when $h(s)=s$, the main operator of the second order in 
\eqref{eq.sch5-2} is unbounded. In order to prove the existence of solutions, 
Liu and Wang etc.\ \cite{LiWW03} defined a change of variable $v=f^{-1}(u)$ 
and used  it to reformulate the equation to a semilinear one, where $f$ is 
defined by ODE: $f'(t)=(1+2f^2(t))^{-1/2}, ~t\in(0,+\infty)$ and 
$f(t)=-f(-t)$, $t\in(-\infty,0)$.
This method can also be found in some papers about such kind of problems 
thereafter, e.g. \cite{CoJe04,OMS07,OMS10}.

For problems with $h(s)=s^{\alpha},\alpha>0$, it is worthy of pointing out that
when $\alpha>1/2$, the number $2^*(2\alpha)=2^*\times2\alpha$ behaves 
like critical exponent for \eqref{eq.sch5-2} (see \cite{LiWa03}), 
while when $0<\alpha\leq1/2$, the critical number is still $2^*$.

Besides the references mentioned above, there are some papers
study such kind of problems with nonlinear terms at critical growth.
In \cite{SiVi10}, Silva and Vieira considered the problem with $h(s)=s$, 
$l(|u|^2)u=K(x)u^{2(2^*)-1}+g(x,u)$, and proved the existence of solutions
 of \eqref{eq.sch5-2}.
In \cite{Moam06}, Moameni studied the problem with $h(s)=s^{\alpha},\alpha>1/2$ 
and $l(u)$ at critical growth under radially symmetric conditions.
Recently, Li and Zhang in \cite{LiZh13} proved the existence of a positive 
solution for the problem that $h(s)=s^{\alpha}$,
$l(s)=s^{(q-2)/2}+s^{(2^*-2)/2}$, where $\alpha>1/2,~2(2\alpha)\leq q<2^*(2\alpha)$.

There are two main difficulties in the study of  problem \eqref{eq.sch6-main}.
The first one is the main operator of the second order is singular in the equation 
provided that $0<\alpha<1/2$.
Another one is caused by the nonlinear term $|u|^{2^*-2}u$ since the Sobolev 
imbedding from $H_0^1(\Omega)$ into $L^{2^*}(\Omega)$ is not compact.

Recently, the authors in \cite{WangY16,WangY17} studied the existence of standing 
waves of \eqref{eq.sch5-1} with $h(s)=s^\alpha,0<\alpha<1/2$ in ${\mathbb{R}^N}$.
We mention that  \eqref{eq.sch6-main} can be deduced from \eqref{eq.sch5-2} by 
choosing $l(s)=s^{(q-2)/2}+s^{(2^*-2)/2}$.
Inspired by \cite{LiZh13}, in this paper, we consider \eqref{eq.sch6-main} 
on bounded domain $\Omega\subset{\mathbb{R}^N}$.


We denote $X:= H_0^1(\Omega)$ endowed with the norm 
$\|u\|^2=\langle u,u\rangle=\int_{\Omega}\nabla u\nabla u\,\mathrm{d}x$.
Let $f(u)=|u|^{q-2}u+|u|^{2^*-2}u$.
We want to find weak solutions to \eqref{eq.sch6-main}. By {\it weak solution},
we mean a function $u$ in $X$ satisfying that,
for all $\varphi\in C_0^{\infty}(\Omega)$, there holds
\begin{equation}\label{def.I'}
\int_{\Omega}\nabla u\nabla\varphi\,\mathrm{d}x
+\kappa\alpha\int_{\Omega}\nabla(|u|^{2\alpha})\nabla(|u|^{2\alpha-2}u\varphi)
\,\mathrm{d}x
 =\int_{\Omega}f(u)\varphi\,\mathrm{d}x.
\end{equation}

According to the variational methods, the weak solutions of \eqref{eq.sch6-main}
corresponds to the critical points
of the functional $I: X\to{\mathbb{R}}$ defined by
\begin{equation}\label{def.I}
I(u)=\frac{1}{2}\int_{\Omega}(1+2\kappa\alpha^2|u|^{2(2\alpha-1)})
|\nabla u|^2\,\mathrm{d}x
-\int_{\Omega}F(u)\,\mathrm{d}x,
\end{equation}
where $F(t)=\int_0^uf(s)\,\mathrm{d}s$. For $u\in X$, $I(u)$ is 
lower semicontinuous when $0<\alpha<1/2$, and not differentiable in all 
directions $\varphi\in X$. To overcome this difficulty, we use a change 
of variable to reformulate functional $I$. This make it possible for us 
to use the classical critical point theorem.

Let $g(t)=(1+2\kappa\alpha^2|t|^{2(2\alpha-1)})^{1/2}$, then $g(t)$ is 
monotone and decreasing in $t\in(0,+\infty)$. Note that for $t_0>0$ 
sufficiently small, we have
\[
\int_0^{t_0}g(s)\,\mathrm{d}s\leq2\alpha\sqrt{\kappa}
\int_0^{t_0}s^{2\alpha-1}\,\mathrm{d}s
=\sqrt{\kappa}t_0^{2\alpha},
\]
thus we can define a function $G:{\mathbb{R}}\to{\mathbb{R}}$ by
\begin{equation}\label{def.h}
v=G(u)=\int_0^ug(s)\,\mathrm{d}s.
\end{equation}
Then $G$ is invertible and odd.

Let $G^{-1}$ be the inverse function of $G$, then 
$\frac{\mathrm{d}}{\mathrm{d}v}G^{-1}(v)\in[0,1)$.
Inserting $u=G^{-1}(v)$ into \eqref{def.I}, we get
\begin{equation}\label{def.J}
J(v):=I(G^{-1}(v))=\frac{1}{2}\int_{\Omega}|\nabla v|^2\,\mathrm{d}x
-\int_{\Omega}F(G^{-1}(v))\,\mathrm{d}x.
\end{equation}
We can prove that (see Proposition \ref{prop.J}) $J$ is well defined on $X$, 
and is continuous in $X$. Moreover, it is also G\^{a}teaux-differentiable, 
and for $\psi\in C_0^\infty(\Omega)$,
\begin{equation}\label{def.J'}
\langle J'(v),\psi\rangle=\int_{\Omega}\nabla v\nabla\psi\,\mathrm{d}x
-\int_{\Omega}\frac{f(G^{-1}(v))}{g(G^{-1}(v))}\psi\,\mathrm{d}x.
\end{equation}


Assume that $v\in X$ with $v>0, x\in\Omega$ and $v=0, x\in\partial\Omega$ 
be such that equality $\langle J'(v),\psi\rangle=0$ holds for all 
$\psi\in C_0^\infty(\Omega)$. Let $u=G^{-1}(v)$, then by \eqref{def.h}, 
$\nabla v=g(u)\nabla u$. Accordingly, $\nabla u=\frac{\nabla v}{g(G^{-1}(v))}$. 
Thus we get $u\in X$.


For  $\varphi\in C_0^\infty(\Omega)$, let $\psi=g(G^{-1}(v))\varphi$, then
$\nabla \psi=g(G^{-1}(v))\nabla\varphi
+\frac{g'(G^{-1}(v))\varphi}{g(G^{-1}(v))}\nabla v$.  Since
\begin{align*}
\nabla v\nabla\psi&= g(G^{-1}(v))\nabla v\nabla\varphi
+\frac{g'(G^{-1}(v))\varphi}{g(G^{-1}(v))}|\nabla v|^2 \\
    &= g^2(u)\nabla u\nabla \varphi+g(u)g'(u)\varphi|\nabla u|^2,
\end{align*}
from \eqref{def.J'}, we obtain that
\[
\int_{\Omega}g^2(u)\nabla u\nabla\varphi
+\int_{\Omega}g(u)g'(u)\varphi|\nabla u|^2
-\int_{\Omega}f(u)\varphi=0.
\]
This implies that $u$ such that \eqref{def.I'} holds.
In summary, to find a weak solution to \eqref{eq.sch6-main}, it suffices to 
find a positive weak solution to the following equation
\begin{equation}\label{eq.sch5-h}
-\Delta v=\frac{f(G^{-1}(v))}{g(G^{-1}(v))},\quad x\in\Omega.
\end{equation}

We assume that
\begin{itemize}
\item[(H1)] assume that $q\in(2,2^*)$ and either 
\begin{itemize}
\item[(i)] $\frac{1}{4}<\alpha<\frac{1}{2}$, $q>\frac{4}{N-2}+4\alpha$ or 
\item[(ii)] $0<\alpha\leq\frac{1}{4}$, $q>\frac{N+2}{N-2}$ holds.
\end{itemize}
\end{itemize}
Note that for $\frac{1}{4}<\alpha<\frac{1}{2}$, we have 
$q>\frac{4}{N-2}+4\alpha>\frac{N+2}{N-2}$.
The following theorem is the main result of this article.

\begin{theorem}\label{thm.m1}
Assume that {\rm (H1)} holds.
Then problem \eqref{eq.sch6-main} has a positive weak solution in $X$.
\end{theorem}

In Section \ref{sec:6-2}, we study the properties of the function $G^{-1}$ and 
show that the functional $J$ has the mountain pass geometry.
In Section \ref{sec:6-3}, we first prove that every Palais-Smale sequence 
$\{v_n\}$ of $J$ is bounded in $X$, then we employ the mountain pass theorem 
to prove the existence of nontrivial solution to \eqref{eq.sch5-h}. 
A crucial step is to prove that the weak limit $v$ of $\{v_n\}$ is nonzero.

In this article, $\|\cdot\|_p$ denotes the norm of Lebesgue space 
$L^p(\Omega)$ and $C_k,~k=1,2,3,\cdots$ will denote positive constants.

\section{Mountain pass geometry} \label{sec:6-2}

The following lemma gives some properties of the transformation $G^{-1}$.

\begin{lemma}\label{lem.h}
The function $G^{-1}(t)$ has the following properties,
\begin{itemize}
  \item[(1)] $G^{-1}(t)$ is odd, invertible, increasing and of class $C^1$ 
for $0<\alpha<1/2$, of class $C^2$ for $0<\alpha<1/4$;
  \item[(2)] $|\frac{\mathrm{d}}{\mathrm{d}t}G^{-1}(t)|\leq1$ for all 
$t\in{\mathbb{R}}$;
  \item[(3)] $|G^{-1}(t)|\leq |t|$ for all $t\in{\mathbb{R}}$;
  \item[(4)] $(G^{-1}(t))^{2\alpha}/t\to\sqrt{2/\kappa}$ as $t\to0^+$;
  \item[(5)] $2\alpha G^{-1}(t)g(G^{-1}(t))\leq 2\alpha t
\leq G^{-1}(t)g(G^{-1}(t))$ for $t>0$;
  \item[(6)] $G^{-1}(t)/t\to1$ as $t\to+\infty$;
\end{itemize}
\end{lemma}

\begin{proof}
For  (1) and (2), $G^{-1}(t)$ is odd and invertible by definition.
 Moreover, $\frac{\mathrm{d}}{\mathrm{d}t}G^{-1}(t)=[g(G^{-1}(t))]^{-1}\in[0,1]$.
Thus $G^{-1}(t)$ is increasing and of class $C^1$
for $0<\alpha<1/2$. By direct computation, we have
\[
\frac{\mathrm{d}^2}{\mathrm{d}t^2}G^{-1}(t)
=2\kappa\alpha^2(1-2\alpha)\frac{|G^{-1}(t)|^{-4\alpha}G^{-1}(t)}
{\big(2\kappa\alpha^2+|G^{-1}(t)|^{2(1-2\alpha)}\big)^2}.
\]
This implies that $G^{-1}(t)$ is of class $C^2$ provided that $0<\alpha<1/4$.

For  (3), assume that $t>0$ and note that $g(G^{-1}(t))>1$, we have
\[ %\label{lem.h.1}
0\leq G^{-1}(t)=\int_0^{G^{-1}(t)}\,\mathrm{d}s
\leq \int_0^{G^{-1}(t)}g(s)\,\mathrm{d}s=t.
\]
Then the conclusion follows since $G^{-1}$ is odd.

For  (4), note that from part (3), we have $G^{-1}(t)\to 0$ as
 $t\to 0$. Thus by employing
L'H\^opital's Rule, we get
\[
\lim_{t\to0^+}\frac{(G^{-1}(t))^{2\alpha}}{t}
=\lim_{t\to0^+}\frac{2\alpha (G^{-1}(t))^{2\alpha-1}}{g(G^{-1}(t))}
=\sqrt{\frac{2}{\kappa}}.
\]

For  (5), we prove the right-hand side inequality. 
Let $H(t)=G^{-1}(t)g(G^{-1}(t))$ and $\tilde{H}(t)=H(t)-2\alpha t$. 
Then $\tilde{H}(0)=0$.
We prove that $\frac{\mathrm{d}}{\mathrm{d}t}\tilde{H}(t)\geq0$, i.e. 
$\frac{\mathrm{d}}{\mathrm{d}t}H(t)\geq 2\alpha$, and this implies the conclusion.
In fact, for $t=0$, by part (4) and note that $G^{-1}(t)$ has same sign of 
$t$, we have
\[
\frac{\mathrm{d}}{\mathrm{d}t}\Big|_{t=0}H(t)
=\lim_{t\to0}\frac{H(t)}{t}
=\lim_{t\to0}\sqrt{\frac{2}{\kappa}}\frac{|H(t)|}{|G^{-1}(t)|^{2\alpha}}
=\sqrt{\frac{2}{\kappa}}\sqrt{2\kappa\alpha^2}=2\alpha.
\]
For $t\neq0$, we have
\begin{align*}
\frac{\mathrm{d}}{\mathrm{d}t}H(t)
&= \frac{\mathrm{d}}{\mathrm{d}t}\Big(\frac{G^{-1}(t)
 \big(2\kappa\alpha^2+|G^{-1}(t)|^{2(1-2\alpha)}\big)^{1/2}}{|G^{-1}(t)|^{1-2\alpha}}
 \Big) \\
&\geq \frac{|G^{-1}(t)|^{2(1-2\alpha)}-(1-2\alpha)|G^{-1}(t)
 |^{2(1-2\alpha)}}{|G^{-1}(t)|^{2(1-2\alpha)}}=2\alpha.
\end{align*}
The left-hand side inequality can be proved similarly.

For part (6), since $\frac{\mathrm{d}}{\mathrm{d}t}G^{-1}(t)>1/2$ for 
$t>0$ sufficiently large, we conclude that $G^{-1}(t)\to+\infty$ as $t\to+\infty$.
Thus by employing L'H\^opital's Rule again, we have 
$\lim_{t\to+\infty}G^{-1}(t)/t=\lim_{t\to+\infty}
\frac{\mathrm{d}}{\mathrm{d}t}G^{-1}(t)=1$.
\end{proof}

By the definition and properties of $G^{-1}$, we have the following imbedding 
results.

\begin{lemma}\label{lem.embed}
The map: $v\to G^{-1}(v)$ from $X$ into $L^p(\Omega)$ is continuous for 
$2\leq p\leq 2^*$, and is compact for $2\leq p<2^*$.
\end{lemma}

The above lemma can be proved by using (2)-(3) of Lemma \ref{lem.h}.
In the next two lemmas, we estimate the remainder of $v-G^{-1}(v)$ at infinity.
The results obtained will be used to compute the mountain pass level in 
the proof of the main theorem.

\begin{lemma}\label{lem.h-2*-1}
There exists $d_0>0$ such that
\[
\lim_{v\to+\infty}(v-G^{-1}(v))\geq d_0.
\]
\end{lemma}

\begin{proof}
Assume that $v>0$. By Lemma \ref{lem.h}, it follows that $G^{-1}(v)\leq v$ and 
$G^{-1}(v)g(G^{-1}(v))\leq v$. Thus we have
\begin{align*}
v-G^{-1}(v)
&\geq v\Big(1-\frac{1}{g(G^{-1}(v))}\Big)\\
&= v\frac{(2\kappa\alpha^2+G^{-1}(v)^{2(1-2\alpha)})^{1/2}-G^{-1}(v)^{1-2\alpha}}
{(2\kappa\alpha^2+G^{-1}(v)^{2(1-2\alpha)})^{1/2}}\\
&\geq \frac{\kappa\alpha^2v}{2\kappa\alpha^2+G^{-1}(v)^{2(1-2\alpha)}}\\
&\geq \frac{\kappa\alpha^2v}{2G^{-1}(v)^{2(1-2\alpha)}}\quad\mbox{for $v$ large}\\
&:= d(\alpha,v).
\end{align*}
\smallskip

\noindent\textbf{Case 1.}
 If $\frac{1}{4}<\alpha<\frac{1}{2}$, then $0<1-2\alpha<1$ and thus 
$d(\alpha,v)\to+\infty$ as $v\to+\infty$.

\noindent\textbf{Case 2.}
 If $\alpha=\frac{1}{4}$, then $1-2\alpha=1$ and thus 
$d(\alpha,v)\to\frac{\kappa\alpha^2}{2}$ as $v\to+\infty$.

\noindent\textbf{Case 3.}
 If $0<\alpha<\frac{1}{4}$, we claim that $v-G^{-1}(v)\to0$ is impossible.
Assume on the contrary. Note that $4\alpha<1$ and $(G^{-1}(v))^{4\alpha-1}\to0$ 
as $v\to+\infty$, by L'H\^opital's Rule, we have
\begin{align*}
0&\leq \lim_{v\to+\infty}\frac{v-G^{-1}(v)}{G^{-1}(v)^{4\alpha-1}}\\
&= \lim_{v\to+\infty}\frac{G^{-1}(v)^{1-2\alpha}}{4\alpha-1}
[(2\kappa\alpha^2+G^{-1}(v)^{2(1-2\alpha)})^{1/2}-G^{-1}(v)^{1-2\alpha}]\\
&= \frac{\kappa\alpha^2}{4\alpha-1}<0,
\end{align*}
a contradiction.
In summary, for all $0<\alpha<1/2$, there exists $d_0>0$ such that the 
conclusion of the lemma holds.
\end{proof}

\begin{lemma}\label{lem.h-2*-2}
For $G^{-1}(v)$ defined in \eqref{def.h}, we have
\begin{itemize}
\item[(i)] If $\frac{1}{4}<\alpha<\frac{1}{2}$, then
\[
\lim_{v\to+\infty}\frac{v-G^{-1}(v)}{v^{4\alpha-1}}
=\frac{\kappa\alpha^2}{4\alpha-1};
\]

\item[(ii)] If $0<\alpha\leq\frac{1}{4}$, then
\[
\lim_{v\to+\infty}\frac{v-G^{-1}(v)}{\log G^{-1}(v)}
\leq   \begin{cases}
     \frac{\kappa}{16}, & \alpha=\frac{1}{4}, \\
     0, & 0<\alpha<\frac{1}{4}.
   \end{cases}
\]
\end{itemize}
\end{lemma}

\begin{proof}
(i) Assume that $\frac{1}{4}<\alpha<\frac{1}{2}$. By the proof of Lemma 
\ref{lem.h-2*-1}, we have $v-G^{-1}(v)\to+\infty$ as $v\to+\infty$. 
Then we can use L'Hopital Principle to get
\[
\lim_{v\to+\infty}\frac{v-G^{-1}(v)}{v^{4\alpha-1}}
=\lim_{v\to+\infty}\frac{g(G^{-1}(v))-1}{(4\alpha-1)v^{4\alpha-2}g(G^{-1}(v))}
=\frac{\kappa\alpha^2}{4\alpha-1}
\]

(ii) Assume that $0<\alpha\leq\frac{1}{4}$. If there exists a constant 
$C>0$ such that $v-G^{-1}(v)\leq C$, then the conclusion holds. Otherwise, 
we may assume that $v-G^{-1}(v)\to+\infty$ as $v\to+\infty$.
 Again by L'Hopital Principle, we have
\begin{align*}
A&:=&\lim_{v\to+\infty}\frac{v-G^{-1}(v)}{\log G^{-1}(v)}\\
&= \lim_{v\to+\infty}G^{-1}(v)\Big(\frac{1}{g(G^{-1}(v))}-1\Big)\\
&= \lim_{v\to+\infty}\frac{2\kappa\alpha^2G^{-1}(v)^{2\alpha}}
{(2\kappa\alpha^2+G^{-1}(v)^{2(1-2\alpha)})^{1/2}+G^{-1}(v)^{1-2\alpha}}.
\end{align*}
Thus $A=\frac{\kappa}{16}$ when $\alpha=\frac{1}{4}$ and $A=0$ when 
$0<\alpha<\frac{1}{4}$.
This completes the proof.
\end{proof}

\section{Proof of main results} \label{sec:6-3}

In this section, we first prove that the functional $J$ is well defined on 
$X$, moreover, it is continuous and G\^{a}teaux-differentiable in $X$; 
next we show that $J$ has the mountain pass geometry, then we use mountain 
pass theorem to prove our main results, this include the construction of 
a path has level $c\in(0,S^{N/2}/N)$.

\begin{proposition}\label{prop.J}
The functional $J$ has the following properties:
\begin{itemize}
\item[(1)] $J$ is well defined on $X$, 
\item[(2)] $J$ is continuous in $X$, 
\item[(3)] $J$ is G\^{a}teaux-differentiable.
\end{itemize}
\end{proposition}

\begin{proof} 
Conclusions (1) and (2) can be proved by using items (2)-(3) of 
Lemma \ref{lem.h} and H\"{o}lder's inequality, we only prove conclusion (3).
Since $G^{-1}\in C^1(\mathbb{R},\mathbb{R})$, for $v\in X$, $t>0$ and 
for any $\psi\in X$, by Mean Value Theorem, there exists 
$\theta\in(0,1)$ such that
\[
\frac{1}{t}\int_{\Omega}\big[F(G^{-1}(v+t\psi))-F(G^{-1}(v))\big]\,\mathrm{d}x
=\int_{\Omega}\frac{f(G^{-1}(v+\theta t\psi))}{g(G^{-1}(v+\theta t\psi))}
\psi\,\mathrm{d}x.
\]
Then by items (2),(3) of Lemma \ref{lem.h}, and Lebesgue's dominated convergence 
theorem, we have
\begin{align*}
& \Big|\int_{\Omega}\frac{f(G^{-1}(v+\theta t\psi))}{g(G^{-1}(v+\theta t\psi))}
 \psi\,\mathrm{d}x
-\int_{\Omega}\frac{f(G^{-1}(v))}{g(G^{-1}(v))}\psi\,\mathrm{d}x\Big| \\
&\leq\int_\Omega\Big|\frac{f(G^{-1}(v+\theta t\psi))}{g(G^{-1}(v+\theta t\psi))}\psi
-\frac{f(G^{-1}(v))}{g(G^{-1}(v+\theta t\psi))}\psi\Big|\,\mathrm{d}x \\
&\quad +\int_\Omega\Big|\frac{f(G^{-1}(v))}{g(G^{-1}(v+\theta t\psi))}\psi
 -\frac{f(G^{-1}(v))}{g(G^{-1}(v))}\psi\Big|\,\mathrm{d}x \\
&\leq\int_{\Omega}\big|f(G^{-1}(v+\theta t\psi))-f(G^{-1}(v))
 \big||\psi|\,\mathrm{d}x \\
&\quad +\int_{\Omega}\big|f(G^{-1}(v))\big|\Big|
 \frac{1}{g(G^{-1}(v+\theta t\psi))}
 -\frac{1}{g(G^{-1}(v))}\Big||\psi|\,\mathrm{d}x \to 0,
\end{align*}
as $t\to0$. Therefore,
\[
\frac{1}{t}\int_{\Omega}\big[F(G^{-1}(v+t\psi))
 -F(G^{-1}(v))\big]\,\mathrm{d}x\to\int_{\Omega}
 \frac{f(G^{-1}(v))}{g(G^{-1}(v))}\psi\,\mathrm{d}x.
\]
This implies that $J$ is G-differentiable.
\end{proof}

\begin{remark} \label{rmk3.2} \rm
Let $v\in X$. Assume that $w\in X$ and $w\to v$. By using similar 
arguments as for Lemma \ref{prop.J}, one can prove that
\[
\langle J'(w)-J'(v),\psi\rangle\to0,\quad\forall \psi\in X.
\]
This means that $J$ is Fr\'{e}chet-differentiable.
\end{remark}

In the following, we consider the existence of positive solutions to 
\eqref{eq.sch5-h}. From variational point of view, non-negative weak 
solutions of the equation correspond to the nontrivial critical points 
of the functional
\[
J^+(v)=\frac{1}{2}\int_{\Omega}|\nabla v|^2\,\mathrm{d}x
-\int_{\Omega}F(G^{-1}(v)^+)\,\mathrm{d}x.
\]
To avoid cumbersome notation, we  denote $J^+(v)$ and $F(G^{-1}(v)^+)$
 by $J(v)$ and $F(G^{-1}(v))$ respectively.

\begin{proposition}\label{prop.mp-1}
There exist $\rho_0, a_0>0$ such that $J(v)\geq a_0$ for all $\|v\|=\rho_0$.
\end{proposition}

\begin{proof}
Note that $|G^{-1}(v)|\leq v$, by Sobolev inequality, we have
\begin{align*}
J(v)&=  \frac{1}{2}\int_{\Omega}|\nabla v|^2\,\mathrm{d}x
-\int_{\Omega}F(G^{-1}(v))\,\mathrm{d}x\\
&\geq  \frac{1}{2}\int_{\Omega}|\nabla v|^2\,\mathrm{d}x
-\frac{1}{q}\int_{\Omega}|v|^q\,\mathrm{d}x
-\frac{1}{2^*}\int_{\Omega}|v|^{2^*}\,\mathrm{d}x\\
&\geq  C_1\|v\|^2-C_2(\|v\|^q+\|v\|^{2^*}).
\end{align*}
Since $2^*>q>2$, there exist $\rho>0$ and $a_0>0$ such that
$J(v)\geq a_0$ for all $\|v\|=\rho$.
\end{proof}

\begin{proposition}\label{prop.mp-2}
There exists $v_0\in X$ with $\|v_0\|>\rho_0$ such that $J(v_0)<0$.
\end{proposition}

\begin{proof}
Let $\varepsilon>0$ be such that 
$\overline{B}_{2\varepsilon}=\{x\in{\mathbb{R}}^N:|x|<2\varepsilon\}\subset\Omega$.
 We take $\varphi\in C_0^{\infty}(\Omega,[0,1])$ with 
$\operatorname{suppt}(\varphi)=\overline{B}_{2\varepsilon}$ and 
$\varphi(x)=1$ for $x\in B_\varepsilon$.
Note that $\lim_{t\to+\infty}G^{-1}(t\varphi)/t\varphi=1$, we have 
$F(G^{-1}(t\varphi))\geq\frac{1}{2}F(t\varphi)$ for $t\in\mathbb{R}$ large enough.
This gives
\begin{align*}
J(t\varphi)\leq\frac{t^2}{2}\int_{\Omega}|\nabla\varphi|^2\,\mathrm{d}x
-\frac{t^q}{2q}\int_{B_\varepsilon}|\varphi|^q\,\mathrm{d}x
-\frac{t^{2^*}}{22^*}\int_{B_\varepsilon}|\varphi|^{2^*}\,\mathrm{d}x
\end{align*}
Choosing $t_0>0$ sufficient large and letting $v_0=t_0\varphi$, we have $J(v_0)<0$.
\end{proof}

As a consequence of Propositions \ref{prop.mp-1}-\ref{prop.mp-2} 
and the Ambrosetti-Rabinowitz Mountain Pass Theorem \cite{Sche99},
there exists a Palais-Smale sequence $\{v_n\}$ of $J$ at level $c$ with
\begin{equation}\label{def.c}
c=\inf_{\gamma\in\Gamma}\sup_{t\in[0,1]}J(\gamma(t))>0,
\end{equation}
where $\Gamma=\{\gamma\in C([0,1],X):\gamma(0)=0,\gamma(1)\neq0,J(\gamma(1))<0\}.$
That is, $J(v_n)\to c,~J'(v_n)\to0$ as $n\to\infty$.

\begin{proposition}\label{prop.ps-1}
Assume that $\{v_n\}$ is a Palais-Smale sequence for $J$, then 
$\{v_n\}$ and $\{G^{-1}(v_n)\}$ are bounded in $X$.
\end{proposition}

\begin{proof}
Since $\{v_n\}\subset X$ is a Palais-Smale sequence, we have
\begin{equation}\label{prop.ps-1-1}
J(v_n)=\frac{1}{2}\int_{\Omega}|\nabla v_n|^2\,\mathrm{d}x
-\int_{\Omega}F(G^{-1}(v_n))\,\mathrm{d}x\to c,
\end{equation}
and for any $\psi\in X$,
\begin{equation}\label{prop.ps-1-2}
\langle J'(v_n),\psi\rangle
= \int_{\Omega}\Big[\nabla v_n\nabla\psi
-\frac{f(G^{-1}(v_n))}{g(G^{-1}(v_n))}\psi\Big]\,\mathrm{d}x=o(1)\|\psi\|.
\end{equation}
Note that $G^{-1}(t)g(G^{-1}(t))\to0$ as $t\to0$, we have 
$G^{-1}(v_n)g(G^{-1}(v_n))\in X$ by direct computation. Thus we can take 
$\psi=G^{-1}(v_n)g(G^{-1}(v_n))$ as test functions and get
\begin{equation}\label{prop.ps-1-3}
\begin{aligned}
\langle J'(v_n),\psi\rangle 
&=  \int_{\Omega}|\nabla v_n|^2\,\mathrm{d}x
 -\int_{\Omega}f(G^{-1}(v_n))G^{-1}(v_n)\,\mathrm{d}x \\
&\quad -\int_{\Omega}\frac{2\kappa\alpha^2(1-2\alpha)}{2\kappa\alpha^2
 +|G^{-1}(v_n)|^{2(1-2\alpha)}}|\nabla v_n|^2\,\mathrm{d}x.
\end{aligned}
\end{equation}
It follows that
\begin{align*}
c+o(1) = J(v_n)-\frac{1}{q}\langle J'(v_n),\psi\rangle
    \geq \big(\frac{1}{2}-\frac{1}{q}\big)
\int_{\Omega}|\nabla v_n|^2\,\mathrm{d}x.
\end{align*}
Since $q>2$, we obtain that $\{v_n\}$ is bounded in $X$. 
Note that $|\nabla G^{-1}(v_n)|^2\leq|\nabla v_n|^2$, we conclude that 
$\{G^{-1}(v_n)\}$ is also bounded in $X$.
\end{proof}

Since $v_n$ is a bounded Palais-Smale sequence, there exists $v\in X$ such 
that $v_n\rightharpoonup v$ in $X$.
Then by Lemma \ref{lem.h} and Lebesgue's dominated convergence theorem, 
for any $\psi\in X$, we have
\begin{align*}
&\langle J'(v_n)-J'(v),\psi\rangle\\
&= \int_{\Omega}(\nabla v_n-\nabla v)\nabla\psi\,\mathrm{d}x\\
&\quad -\int_{\Omega}\Big(\frac{|G^{-1}(v_n)|^{q-2}G^{-1}(v_n)}{g(G^{-1}(v_n))}
 -\frac{|G^{-1}(v)|^{q-2}G^{-1}(v)}{g(G^{-1}(v))}\Big)\psi\,\mathrm{d}x\\
&\quad -\int_{\Omega}\Big(\frac{|G^{-1}(v_n)|^{2^*-2}G^{-1}(v_n)}{g(G^{-1}(v_n))}
 -\frac{|G^{-1}(v)|^{2^*-2}G^{-1}(v)}{g(G^{-1}(v))}\Big)\psi\,\mathrm{d}x
\to0.
\end{align*}
Note that $\langle J'(v_n),\psi\rangle\to0$, we get $J'(v)=0$.
This means that $v$ is a weak solution of \eqref{eq.sch6-main}.
 Now we show that $v$ is nontrivial.


\begin{proposition}\label{prop.ps-3}
Let $\{v_n\}$ be a Palais-Smale sequence for functional $J$ at  level 
$c\in (0,\frac{1}{N}S^{N/2})$, assume that $v_n\rightharpoonup v$ in $X$, 
then $v\neq0$.
\end{proposition}

\begin{proof}
We prove the proposition by contradiction. Assume that $v=0$.
Let $\psi=G^{-1}(v_n)g(G^{-1}(v_n))$. Reasoning as for \eqref{prop.ps-1-3}, we get
\begin{align*}
\langle J'(v_n),\psi\rangle
&= \int_{\Omega}\frac{4\kappa\alpha^3+|G^{-1}(v_n)|^{2(1-2\alpha)}}
 {2\kappa\alpha^2+|G^{-1}(v_n)|^{2(1-2\alpha)}}|\nabla v_n|^2\,\mathrm{d}x
 -\int_{\Omega}f(G^{-1}(v_n))G^{-1}(v_n)\,\mathrm{d}x\\
&\geq \int_{\Omega}\frac{|G^{-1}(v_n)|^{2(1-2\alpha)}}{2\kappa\alpha^2
 +|G^{-1}(v_n)|^{2(1-2\alpha)}}|\nabla v_n|^2\,\mathrm{d}x
 -\int_{\Omega}f(G^{-1}(v_n))G^{-1}(v_n)\,\mathrm{d}x\\
&= \int_{\Omega}|\nabla G^{-1}(v_n)|^2\,\mathrm{d}x
 -\int_{\Omega}f(G^{-1}(v_n))G^{-1}(v_n)\,\mathrm{d}x.
\end{align*}
As the term $|G^{-1}(v_n)|^{q}$ is subcritical, we infer from 
$\langle J'(v_n),G^{-1}(v_n)g(G^{-1}(v_n))\rangle=o(1)$ that
\[
o(1)\geq\|G^{-1}(v_n)\|^2-\|G^{-1}(v_n)\|_{2^*}^{2^*}.
\]
By Sobolev inequality, we have $\|u\|^2\geq S\|u\|_{2^*}^2$ for all $u\in X$,
 where $S$ is the best constant for the imbedding 
$H_0^1(\Omega)\hookrightarrow L^{2^*}(\Omega)$; then we obtain
\begin{align*}
o(1)\geq \|G^{-1}(v_n)\|^2(1-S^{-2^*/2}\|G^{-1}(v_n)\|^{2^*-2}).
\end{align*}
Assume that $\|G^{-1}(v_n)\|\to0$, then by Sobolev inequality, 
we have $\|G^{-1}(v_n)\|_r\to0,~\forall  r\in[2,2^*]$.
Using (5) of Lemma \ref{lem.h}, we conclude that
\begin{align*}
\int_{{\mathbb{R}}^N}|\nabla v_n|^2\,\mathrm{d}x
&= \langle J'(v_n),v_n\rangle +\int_{{\mathbb{R}}^N}
 \frac{|G^{-1}(v_n)|^{q-2}G^{-1}(v_n)}{g(G^{-1}(v_n))}v_n\,\mathrm{d}x\\
&\quad +\int_{{\mathbb{R}}^N}\frac{|G^{-1}(v_n)|^{2^*-2}
 G^{-1}(v_n)}{g(G^{-1}(v_n))}v_n\,\mathrm{d}x\\
&\leq \langle J'(v_n),v_n\rangle +\frac{1}{2\alpha}
 \int_{{\mathbb{R}}^N}|G^{-1}(v_n)|^{q}\,\mathrm{d}x
 +\frac{1}{2\alpha}\int_{{\mathbb{R}}^N}|G^{-1}(v_n)|^{2^*}\,\mathrm{d}x\\
&\to 0,
\end{align*}
This contradicts $J(v_n)\to c>0$; therefore
\[
\|G^{-1}(v_n)\|_{2^*}^{2^*}\geq S^{N/2}+o(1).
\]
Again by (5) of Lemma \ref{lem.h}, we have
\begin{align*}
c&= \lim_{n\to\infty}\Big\{J(v_n)-\frac{1}{2}\langle J'(v_n),v_n\rangle\Big\}\\
&= \lim_{n\to\infty}\Big\{\int_{{\mathbb{R}}^N}|G^{-1}(v_n)|^{q-2}
 \Big(\frac{1}{2}\frac{G^{-1}(v_n)v_n}{g(G^{-1}(v_n))}
 -\frac{1}{q}G^{-1}(v_n)^2\Big)\,\mathrm{d}x\\
&\quad +\int_{{\mathbb{R}}^N}|G^{-1}(v_n)|^{2^*-2}
 \Big(\frac{1}{2}\frac{G^{-1}(v_n)v_n}{g(G^{-1}(v_n))}
 -\frac{1}{2^*}G^{-1}(v_n)^2\Big)\,\mathrm{d}x\Big\}\\
&\geq \lim_{n\to\infty}\Big(\frac{1}{2}-\frac{1}{2^*}\Big)
 \int_{{\mathbb{R}}^N}|G^{-1}(v_n)|^{2^*}\,\mathrm{d}x\\
&\geq \frac{1}{N}S^{N/2}
\end{align*}
which contradicts $c<\frac{1}{N}S^{N/2}$. 
Thus we conclude that $\{v_n\}$ does not vanish.
\end{proof}

Next, we construct a path which minimax level is less than 
$\frac{1}{N}S^{N/2}$ and prove Theorem \ref{thm.m1}.
We follow the strategy used in \cite{BrNi93}.

\begin{proposition}\label{prop.mp-3}
The minimax level $c$ defined in \eqref{def.c} satisfies
$c<\frac{1}{N}S^{N/2}$.
\end{proposition}

\begin{proof}
Let
\[
v^*=\frac{[N(N-2)\varepsilon^2]^{(N-2)/4}}{(\varepsilon^2+|x|^2)^{(N-2)/2}}
\]
be the solution of $-\Delta u=u^{2^*-1}$ in $\mathbb{R}^N$. Then
\begin{align*}
\int_{{\mathbb{R}}^N}|\nabla v^*|^2\,\mathrm{d}x
=\int_{{\mathbb{R}}^N}|v^*|^{2^*}\,\mathrm{d}x=S^{N/2},
\end{align*}
Let $\eta_\varepsilon(x)\in C_0^\infty(\Omega,[0,1])$ be a cut-off function with
$\eta_\varepsilon(x)=1$ in $B_\varepsilon=\{x\in{\Omega}:|x|\leq\varepsilon\}$ and
$\eta_\varepsilon(x)=0$ in $B^c_{2\varepsilon}={\Omega}\setminus B_{2\varepsilon}$.
Let $v_\varepsilon=\eta_\varepsilon v^*$.
For any $\varepsilon>0$, there exists $t^\varepsilon>0$ such that 
$J(t^\varepsilon v_\varepsilon)<0$ for all $t>t^\varepsilon$.
 Define the class of paths
\begin{align*}
\Gamma_\varepsilon=\{\gamma\in C([0,1],X):\gamma(0)=0,\gamma(1)
=t^\varepsilon v_\varepsilon\}
\end{align*}
and the minimax level
\begin{align*}
c_\varepsilon=\inf_{\gamma\in\Gamma_\varepsilon}\max_{t\in[0,1]}J(\gamma(t))
\end{align*}
Let $t_\varepsilon$ be such that
\[
J(t_\varepsilon v_\varepsilon)=\max_{t\geq0}J(tv_\varepsilon)
\]
Note that the sequence $\{v_\varepsilon\}$ is uniformly bounded in $X$, 
we conclude that $\{t_\varepsilon\}$ is upper and lower bounded by two
 positive constants. In fact, if $t_\varepsilon\to0$, we have
 $J(t_\varepsilon v_\varepsilon)\to0$; otherwise, if $t_\varepsilon\to+\infty$,
 we have $J(t_\varepsilon v_\varepsilon)\to-\infty$. In both cases we get 
contradictions according to Proposition \ref{prop.mp-1}. 
This proves the conclusion.

According to \cite{BrNi93}, we have, as $\varepsilon\to0$,
\begin{equation}\label{prop.v-1}
\|\nabla v_\varepsilon\|^2_{2}=S^{N/2}+O(\varepsilon^{N-2}),\quad
\|v_\varepsilon\|^{2^*}_{{2^*}}=S^{N/2}+O(\varepsilon^N).
\end{equation}

We define
\begin{align*}
H(t_\varepsilon v_\varepsilon)
=-\frac{1}{q}\int_{\Omega}G^{-1}(t_\varepsilon v_\varepsilon)^q\,\mathrm{d}x
+\frac{1}{2^*}\int_{\Omega}[(t_\varepsilon v_\varepsilon)^{2^*}
-G^{-1}(t_\varepsilon v_\varepsilon)^{2^*}]\,\mathrm{d}x.
\end{align*}
By the definition of $v_\varepsilon$, for $x\in B_\varepsilon$, there exist 
two constants $c_2\geq c_1>0$ such that for $\varepsilon$ small enough,
\[
c_1\varepsilon^{-(N-2)/2}\leq v_\varepsilon(x)\leq c_2\varepsilon^{-(N-2)/2}
\]
and by (6) of Lemma \ref{lem.h},
\[
c_1\varepsilon^{-(N-2)/2}\leq G^{-1}(v_\varepsilon(x))
\leq c_2\varepsilon^{-(N-2)/2}.
\]
Note that $t_\varepsilon$ is upper and lower bounded, there exists a 
constant $C_1>0$ such that
\begin{equation}\label{prop.mp-1-3}
\int_{B_\varepsilon}G^{-1}(t_\varepsilon v_\varepsilon)^q\,\mathrm{d}x
\geq C_1\varepsilon^{N-q\frac{N-2}{2}} 
=C_1\varepsilon^{(\frac{2^*}{2}-\frac{q}{2})(N-2)}.
\end{equation}
Moreover, since $G^{-1}(t_\varepsilon v_\varepsilon)
\leq t_\varepsilon v_\varepsilon$ and $2^*>2$, by H\"{o}lder inequality, 
we have
\begin{align*}
R_\varepsilon
:=& \frac{1}{2^*}\int_{B_\varepsilon}[(t_\varepsilon v_\varepsilon)^{2^*}
 -G^{-1}(t_\varepsilon v_\varepsilon)^{2^*}]\,\mathrm{d}x\\
\leq& \int_{B_\varepsilon}(t_\varepsilon v_\varepsilon)^{2^*-1}
 (t_\varepsilon v_\varepsilon-G^{-1}(t_\varepsilon v_\varepsilon))\,\mathrm{d}x\\
\leq&\Big(\int_{B_\varepsilon}(t_\varepsilon v_\varepsilon)^{2^*}
 \,\mathrm{d}x\Big)^{\frac{2^*-1}{2^*}} 
 \Big(\int_{B_\varepsilon}(t_\varepsilon v_\varepsilon-G^{-1}
 (t_\varepsilon v_\varepsilon))^{2^*}\,\mathrm{d}x\Big)^{\frac{1}{2^*}}.
\end{align*}
According to Lemma \ref{lem.h-2*-2}, there exists $C_2>0$ such that for 
$\frac{1}{4}<\alpha<\frac{1}{2}$,
\begin{equation}\label{prop.mp-1-4}
R_\varepsilon\leq C_2\Big(\int_{B_\varepsilon}
(t_\varepsilon v_\varepsilon)^{2^*(4\alpha-1)}\,\mathrm{d}x\Big)^{\frac{1}{2^*}}
\leq C_2\varepsilon^{(1-2\alpha)(N-2)};
\end{equation}
while for $0<\alpha\leq\frac{1}{4}$, there exists a constant $\delta\in(0,1)$ 
such that
\begin{equation}\label{prop.mp-1-5}
R_\varepsilon\leq C_2\Big(\int_{B_\varepsilon}
(t_\varepsilon v_\varepsilon)^{2^*\delta}\,\mathrm{d}x\Big)^{\frac{1}{2^*}}
\leq C_2\varepsilon^{\frac{1}{2}(1-\delta)(N-2)}.
\end{equation}
From the above estimations \eqref{prop.mp-1-3}-\eqref{prop.mp-1-5}, we get
\begin{equation}\label{prop.mp-1-6}
H(t_\varepsilon v_\varepsilon)
\leq -C_1\varepsilon^{(\frac{2^*}{2}-\frac{q}{2})(N-2)} 
+C_2\varepsilon^{(1-2\alpha)(N-2)}
\end{equation}
when $\frac{1}{4}<\alpha<\frac{1}{2}$ and
\begin{equation}\label{prop.mp-1-7}
H(t_\varepsilon v_\varepsilon)\
leq -C_1\varepsilon^{(\frac{2^*}{2}-\frac{q}{2})(N-2)}
+C_2\varepsilon^{\frac{1}{2}(1-\delta)(N-2)}
\end{equation}
when $0<\alpha\leq 1/4$.

Now we have
\begin{equation}\label{prop.mp-1-1}
J(t_\varepsilon v_\varepsilon)
=\frac{t_\varepsilon^2}{2}\int_{\Omega}|\nabla v_\varepsilon|^2
-\frac{t_\varepsilon^{2^*}}{2^*}\int_{\Omega}|v_\varepsilon|^{2^*} 
+H(t_\varepsilon v_\varepsilon).
\end{equation}
Since the function $\xi(t)=\frac{1}{2}t^2-\frac{1}{2^*}t^{2^*}$ achieves its 
maximum $\frac{1}{N}$ at point $t_0=1$, by using \eqref{prop.v-1}, 
we derive from \eqref{prop.mp-1-1} that
\begin{equation}\label{prop.mp-1-8}
J(t_\varepsilon v_\varepsilon)\leq \frac{1}{N}S^{N/2}
+H(t_\varepsilon v_\varepsilon)+O(\varepsilon^{N-2}).
\end{equation}
From assumption (H1), we conclude that
\begin{itemize}
\item[(i)] for $\frac{1}{4}<\alpha<\frac{1}{2}$ and $q>\frac{4}{N-2}+4\alpha$, 
 we have $(\frac{2^*}{2}-\frac{q}{2})(N-2)<(1-2\alpha)(N-2)$;

\item[(ii)] for $0<\alpha\leq\frac{1}{4}$ and $q>\frac{N+2}{N-2}$, we have 
 $(\frac{2^*}{2}-\frac{q}{2})(N-2)<\frac{1}{2}(1-\delta)(N-2)$ for 
 $\delta>0$ small enough.
\end{itemize}
Combining \eqref{prop.mp-1-6}, \eqref{prop.mp-1-7} and \eqref{prop.mp-1-8} 
and according to conclusions (i),(ii), we get
\begin{equation}\label{prop.mp-1-9}
c_\varepsilon=J(t_\varepsilon v_\varepsilon)< \frac{1}{N}S^{N/2}.
\end{equation}
Finally, since $\Gamma_\varepsilon\subset\Gamma$, we have
\[
c\leq c_\varepsilon<\frac{1}{N}S^{N/2}.
\]
This completes the proof.
\end{proof}


\begin{proof}[Proof of Theorem \ref{thm.m1}]
Firstly, by Propositions \ref{prop.mp-1}-\ref{prop.mp-2}, the functional 
$J$ has the Mountain Pass Geometry. Then there exists a Palais-Smale sequence 
$\{v_n\}$ at level $c$ given in \eqref{def.c}.
 Secondly, by Proposition \ref{prop.ps-1}, the Palais-Smale sequence 
$\{v_n\}$ is bounded in $X$. By Proposition \ref{prop.ps-3}, 
if $c<\frac{1}{N}S^{N/2}$, then the weak limit $v$ of $\{v_n\}$ in $X$ 
is nonzero and is a critical point of $J$. Finally, by 
Proposition \ref{prop.mp-3}, there indeed exists a mountain pass which 
maximum level $c_\varepsilon$ is strictly less than 
$\frac{1}{N}S^{N/2}$. This implies that the level $c<\frac{1}{N}S^{N/2}$ 
and $v$ is a nontrivial weak solution of Eq.\eqref{eq.sch5-h}.
By strong maximum principle, $v(x)>0,x\in\Omega$. 
Let $u=G^{-1}(v)$. Since $|\nabla u|\leq |\nabla v|$, we obtain that 
$u\in X$ and it is a positive weak solution of \eqref{eq.sch6-main}.
\end{proof}



\subsection*{Acknowledgements} 
The authors want to thank the anonymous referees for their careful reading 
and useful comments.

\begin{thebibliography}{00}

\bibitem{A-M-S} C. O. Alves, D. C. de Morais Filho,  M. A. S. Souto;
  Radially symmetric solutions for a class of critical exponent elliptic 
problems in $\mathbb{R}^N$, {\it Elec. J.  Diff. Equa.}, \textbf{1996(7)} (1996), 1--12.

\bibitem{A-D-S} C. O. Alves, D. C. de Morais Filho,  M. A. S. Souto;
 Multiplicity of positive solutions for a class of problems with critical 
growth in $\mathbb{R}^N$, {\it Proc. Edinb. Math. Soc.}, \textbf{52(1)} (2009), 1--21.

\bibitem{BoGa93} A. V. Borovskii, A. L. Galkin;
 Dynamical modulation of an ultrashort high-intensity laser pulse in matter, 
{\it JETP}, \textbf{77} (1993), 562-573.

\bibitem{BrNi93} H. Brezis, L. Nirenberg;
 Positive solutions of nonlinear elliptic equations involving critical 
Sobolev exponent, {\it Comm. Pure Appl. Math.}, \textbf{36} (1993), 437-477.

\bibitem{CoJe04} M. Colin, L. Jeanjean;
 Solutions for a quasilinear Schr\"{o}dinger equations: a dual approach, 
{\it Nonlinear Anal. TMA}, \textbf{56} (2004), 213-226.

\bibitem{OMS07} J. M. B. do \'{O}, O. H. Miyagaki, S. H. M. Soares;
 Soliton solutions for quasilinear Schr\"{o}dinger equations: the critical 
exponential case, {\it Nonlinear Anal. TMA}, \textbf{67} (2007), 3357-3372.

\bibitem{OMS10} J. M. B. do \'{O}, O. H. Miyagaki, S. H. M. Soares;
 Soliton solutions for quasilinear Schr\"{o}dinger equations with critical growth, 
{\it J. Differential Equations}, \textbf{1248} (2010),   722--744.

\bibitem{Hass80} R. W. Hasse;
 A general method for the solution of nonlinear soliton andkink Schr\"{o}dinger 
equations, {\it Z. Phys.}, \textbf{37} (1980), 83-87.

\bibitem{KoIK90} A. M. Kosevich, B. A. Ivanov, A. S. Kovalev;
 Magnetic solitons, {\it Phys. Rep.}, \textbf{194} (1990), 117-238.

\bibitem{Kuri81} S. Kurihura;
 Large-amplitude quasi-solitons in superfluid films, {\it J. Phys. Soc. Japan}, 
\textbf{50} (1981), 3262-3267.

\bibitem{LiZh13} Z. Li, Y. Zhang;
 Solutions for a class of quasilinear Schrodinger equations with critical 
exponents term, {\it J. Math. Phys.}, \textbf{58} 021501 (2017).

\bibitem{LiSe78} A. G. Litvak, A. M. Sergeev;
 One dimensional collapse of plasma waves, {\it JETP Lett.}, 
\textbf{27} (1978), 517-520.

\bibitem{LiWa03} J. Liu, Z.-Q. Wang;
 Soliton solutions for quasilinear Schr\"{o}dinger equations, I,
 {\it Proc. Amer. Math. Soc.}, \textbf{131} (2003), 441-448.

\bibitem{LiWW03} J. Liu, Y. Wang, Z.-Q. Wang;
 Soliton solutions for quasilinear Schr\"{o}dinger equations, II, 
{\it J. Differential Equations}, \textbf{187} (2003), 473-493.

\bibitem{MaFe84} V. G. Makhankov, V. K. Fedyanin;
 Non-linear effects in quasi-one-dimensional models of condensed matter theory, 
{\it Phys. Rep.}, \textbf{104} (1984), 1-86.

\bibitem{Moam06} A. Moameni;
 Existence of soliton solutions for a quasilinear Schr\"{o}dinger equation 
involving critical exponent in $\mathbb{R}^N$, 
{\it J. Differential Equations,} \textbf{229} (2006), 570-587.

\bibitem{QuCa82} G. R. W. Quispel, H. W. Capel;
 Equation of motion for the Heisenberg spin chain, 
{\it Physica A}, \textbf{110} (1982),  41-80.

\bibitem{Sche99} M. Schechter;
 Linking Methods in Critical Point Theory, Birkh\"{a}user, Boston, 1999.

\bibitem{SiVi10} E. A. B. Silva, G. F. Vieira;
Quasilinear asymptotically periodic Schr\"{o}dinger equations with critical growth,
{\it Calc. Var. PDEs}, \textbf{39} (2010), 1-33.

\bibitem{WangY16} Y. Wang;
 Multiplicity of solutions for singular quasilinear Schrodinger
equations with critical exponents, J. Math. Anal. Appl., \textbf{458} (2) (2018),
1027-1043.

\bibitem{WangY17} Y. Wang, Z. Li, A. A. Abdelgadir;
 On singular quasilinear Schrodinger equations with critical exponents,
Math. Meth. Appl. Sci., \textbf{40} (2017), 5095-5108.



\end{thebibliography}

\end{document}