\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2018 (2018), No. 97, pp. 1--13.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu} \thanks{\copyright 2018 Texas State University.} \vspace{8mm}} \begin{document} \title[\hfilneg EJDE-2018/97\hfil Sublinear fractional Schr\"odinger-type equations] {Infinitely many solutions for sublinear fractional Schr\"odinger-type equations with general potentials} \author[G.-L. Hou, B. Ge, J.-F. Lu \hfil EJDE-2018/97\hfilneg] {Gang-Ling Hou, Bin Ge, Jian-Fang Lu} \address{Gang-Ling Hou \newline College of Aerospace and Civil Engineering, Harbin Engineering University, Harbin, 150001, China} \email{hougl@hrbeu.edu.cn} \address{Bin Ge (corresponding author) \newline Department of Applied Mathematics, Harbin Engineering University, Harbin, 150001, China} \email{gebin791025@hrbeu.edu.cn} \address{Jian-Fang Lu \newline Department of Applied Mathematics, Harbin Engineering University, Harbin, 150001, China} \email{1176678630@qq.com} \dedicatory{Communicated by Vicentiu D. Radulescu} \thanks{Submitted January 27, 2018. Published April 24, 2018.} \subjclass[2010]{26A33, 35J60, 47J30} \keywords{Fractional Laplacian; variational method; sublinear; genus} \begin{abstract} This article concerns the fractional Schr\"odinger type equations $$ (-\Delta)^\alpha u+V(x)u =f(x,u) \quad\text{in } \mathbb{R}^N, $$ where $N\geq 2$, $\alpha\in(0,1)$, $(-\Delta)^\alpha$ stands for the fractional Laplacian, $V$ is a positive continuous potential, $f\in C(\mathbb{R}^N\times\mathbb{R},\mathbb{R})$. We establish criteria that guarantee the existence of infinitely many solutions by using the genus properties in critical point theory. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{definition}[theorem]{Definition} \allowdisplaybreaks \section{Introduction} \label{intro} In this article, we consider the nonlinear Schr\"odinger-type equation \begin{equation} \label{eP} (-\Delta)^\alpha u+V(x)u =f(x,u)\quad\text{in } \mathbb{R}^N, \end{equation} where $N\geq 2$, $\alpha\in(0,1)$, $(-\Delta)^\alpha$ stands for the fractional Laplacian, $V$ is a positive continuous potential, $f\in C(\mathbb{R}^N\times\mathbb{R},\mathbb{R})$. The fractional Laplacian $(-\Delta)^\alpha$ with $\alpha\in(0,1)$ of a function $\phi\in \mathcal {S}$ is defined by $$ \mathcal{F}(((-\Delta)^\alpha)\phi)(\xi)=|\xi|^{2\alpha}\mathcal{F}(\phi)(\xi),\quad \forall \alpha\in(0,1), $$ where $\mathcal {S}$ denotes the Schwartz space of rapidly decreasing $C^\infty$ functions in $\mathbb{R}^N$, $\mathcal {F}$ is the Fourier transform, i.e., $$ \mathcal {F}(\phi)(\xi)=\frac{1}{(2\pi)^{N/2}}\int_{\mathbb{R}^N}e^{-2\pi i\xi\cdot x}\phi(x)dx. $$ If $\phi$ is smooth enough, it can also be computed by the following singular integral $$ (-\Delta)^{\alpha}\phi(x)=c_{N,\alpha}\operatorname{P.V.} \int_{\mathbb{R}^N}\frac{\phi(x)-\phi(y)}{|x-y|^{N+2\alpha}}dy. $$ Here $\operatorname{P.V.}$ is the principal value and $c_{N,\alpha}$ is a normalization constant. The fractional Schr\"odinger equation is a fundamental equation of fractional quantum mechanics. It was discovered by Laskin \cite{1,2} as a result of extending the Feynman path integral, from the Brownian-like to L\'{e}vy-like quantum mechanical paths, where the Feynman path integral leads to the classical Schr\"odinger equation, and the path integral over L\'{e}vy trajectories leads to the fractional Schr\"odinger equation. The study of the fractional Schr\"odinger equations and the corresponding variational problems has received more and more interest in recent years. For example, \cite{128,124,125} studied fractional elliptic problems with critical growth, \cite{8,129} gave some sufficient conditions for the existence of positive solutions to fractional elliptic equation, \cite{3,7,4,5} studied the existence of ground state solutions on $\mathbb{R}^N$ and \cite{126} studied fractional Kirchhoff equations. For more results about the fractional Schr\"odinger equations, we refer to \cite{6,123,130,127,9,10}. It is well known, the main difficulty in treating problem \eqref{eP} in $\mathbb{R}^N$ arises from the lack of compactness of the Sobolev embeddings, which prevents from checking directly that the energy functional associated with \eqref{eP} satisfies the $C$-condition. To overcome the difficulty of the noncompact embedding, Teng \cite{11}, Xu-Wei-Dong \cite{12}, Chen \cite{121}, Bisci-Radulescu \cite{122}, also establish a new compact embedding theorems for the subspace of $H^\alpha(\mathbb{R}^N)$. Furthermore, the authors able to guarantee the existence and multiplicity of nontrivial weak solutions of \eqref{eP} in $E=\{u\in H^\alpha(\mathbb{R}^N):\int_{\mathbb{R}^N}| (-\Delta)^{\alpha/2}u(x)|^{2}dx +\int_{\mathbb{R}^N}V(x)u^2dx <+\infty\}$ provided $\inf V>0$ and the following conditions hold: \begin{itemize} \item[(A1)] For any $M>0$, there exists $r_0>0$ such that \begin{align*} \lim_{|y|\to\infty}\mu(\{x\in\mathbb{R}^N: |x-y|\leq r_0, V(x)\leq M\})=0, \end{align*} where $\mu$ is the Lebesgue measure on $\mathbb{R}^N$. \end{itemize} We emphasize that in our approach, no coerciveness hypothesis (A1) and not necessarily radially symmetric will be required on the potential $V$. To the best of our knowledge, few works concerning on this case up to now. Inspired by the above facts and aforementioned papers, the main purpose of this paper is to study the existence of infinitely many solutions for \eqref{eP} when $F(x,u)$ satisfies sublinear in $u$ at infinity. Our tool used here is the genus properties in critical point theory. Before stating our main results, we first make some assumptions on the functions $V$ and $f$. For the potential $V$, we make the following assumption \begin{itemize} \item[(A2)] $V\in C(\mathbb{R}^N)$ and $V_0:=\inf_{x\in\mathbb{R}^N} V(x)>0$. \end{itemize} For the nonlinearity $f$, we suppose it satisfies the following conditions: \begin{itemize} \item[(A3)] (1) $f\in C(\mathbb{R}^N\times\mathbb{R},\mathbb{R})$ and there exist constant $10$ and $\theta\in(1,2)$ such that $$ F(x,t)\geq \rho|t|^{\theta},\;\forall (x,t)\in I\times [-\delta,\delta], $$ where $F(x,t)=\int_0^t f(x,s)ds$. \end{itemize} The same problem is studied by Shi and Chen \cite{122}. The authors established the existence of at least $k$ distinct pairs of solutions for\eqref{eP} by using the Clark theorem. Inspired by the above-mentioned papers, we study problem \eqref{eP} in the different method. More precisely, the aim of this work is to prove the existence of infinitely many solutions by using the genus properties in critical point theory. We are now in the position to state our main results. \begin{theorem}\label{thm1.1} Suppose that {\rm (A2)} and {\rm (A3)} hold. Then \eqref{eP} possesses at least one nontrivial solution. \end{theorem} \begin{theorem}\label{thm1.2} Suppose that {\rm (A2)} and {\rm (A3)} hold, and $f$ satisfies $$ f(x,-t)=-f(x,t),\;\forall (x,t)\in \mathbb{R}^N\times\mathbb{R}. $$ Then \eqref{eP} possesses infinitely many solutions. \end{theorem} The rest of this article is organized as follows. In Section 2, we state and prove some preliminary results that will be used later. We will finish the proof of our main result (Theorem \ref{thm1.1} and Theorem \ref{thm1.2}) in Section 3. \section{Preliminaries} In this section we recall some results on Sobolev spaces of fractional order. A very complete introduction to fractional Sobolev spaces can be found in \cite{13}. Consider the fractional order Sobolev space $$ H^\alpha(\mathbb{R}^N)=\big\{u\in L^2(\mathbb{R}^N): \int_{\mathbb{R}^N}\big(|\xi|^{2\alpha}\hat{u}^2 +\hat{u}^2\big)d\xi<+\infty\big\}, $$ where $\hat{u}\doteq \mathcal {F}(u)$. The norm is defined by $$ \|u\|_{H^\alpha(\mathbb{R}^N)}=\Big( \int_{\mathbb{R}^N}(|\xi|^{2\alpha}\hat{u}^2+\hat{u}^2)d\xi\Big)^{1/2}. $$ \indent In this paper we consider its subspace: $$ E=\big\{u\in H^\alpha(\mathbb{R}^N): \int_{\mathbb{R}^N}V(x)u^2dx<+\infty\big\} $$ with the norm $$ \|u\|_{E}=\Big( \int_{\mathbb{R}^N}(|\xi|^{2\alpha}\hat{u}^2+\hat{u}^2)d\xi +\int_{\mathbb{R}^N}V(x)u^2dx\Big)^{1/2}. $$ Note that, by Plancherel's theorem we have $|\hat{u}|_2=|u|_2$ and \begin{align*} \int_{\mathbb{R}^N}|(-\Delta)^{\alpha/2}u(x)|^{2}dx =&\int_{\mathbb{R}^N}(\widehat{(-\Delta)^{\alpha/2}u(\xi)})^{2}d\xi\\ =&\int_{\mathbb{R}^N}(|\xi|^{\alpha}\hat{u}(\xi))^{2}d\xi\\ =&\int_{\mathbb{R}^N}|\xi|^{2\alpha}\hat{u}^{2}d\xi <+\infty, \; \forall u\in H^\alpha(\mathbb{R}^N). \end{align*} Together with (A2), it follows that the norm $\|\cdot\|_E$ is equivalent to the norm $$ \|u\|=\Big(\int_{\mathbb{R}^N}|(-\Delta)^{\alpha/2}u(x)|^{2}dx +\int_{\mathbb{R}^N}V(x)u^2dx\Big)^{1/2}. $$ Throughout out this paper, we will use the norm $\|u\|$ in $E$. \begin{lemma}[\cite{8}] \label{lem2.1} $H^\alpha(\mathbb{R}^N)$ continuously embedded into $L^p(\mathbb{R}^N)$ for $p\in[2,2_\alpha^*]$, and compactly embedded into $L_{\rm loc}^p(\mathbb{R}^N)$ for $p\in[2,2_\alpha^*)$. \end{lemma} \begin{lemma} \label{lem2.2} Assume that {\rm (A2), (A3)} hold. Then the functional $\varphi: E \to \mathbb{R}$ defined by \begin{equation}\label{2.1} \varphi(u)=\frac{1}{2}\int_{\mathbb{R}^N}\Big[|(-\Delta)^{\alpha/2} u(x)|^{2}+V(x)u^2\Big]dx-\int_{\mathbb{R}^N}F(x,u)dx \end{equation} is well defined and of class $C^1(E,\mathbb{R})$ and \begin{equation}\label{2.2} \langle\varphi'(u),v\rangle=\int_{\mathbb{R}^N}\Big[ (-\Delta)^{\alpha/2}u(-\Delta)^{\alpha/2}v+V(x)uv\Big]dx -\int_{\mathbb{R}^N}f(x,u)v\,dx. \end{equation} Moreover, the critical points of $\varphi$ in $E$ are solutions of problem \eqref{eP}. \end{lemma} \begin{proof} The functional $\varphi$ is well defined on $E$. Indeed, by virtue of (A3)(1) and the Mean Value Theorem, we have \begin{equation}\label{2.3} F(x,t)\leq \frac{a(x)}{r}|t|^{r}, \quad\forall (x,t)\in\mathbb{R}^N\times\mathbb{R}. \end{equation} For any $u\in E$, we obtain for (A2), \eqref{2.3}, and H\"older inequality that \begin{equation}\label{2.4} \begin{aligned} \int_{\mathbb{R}^N}|F(x,u)|dx \leq&\int_{\mathbb{R}^N}\frac{a(x)}{r}|u|^{r}dx\\ \leq & \int_{\mathbb{R}^N} a(x) |u|^{r}dx\\ =& \int_{\mathbb{R}^N} \frac{a(x)}{V(x)^{r/2}} V(x)^{r/2}|u|^{r}dx\\ \leq& \frac{1}{V_0^{r/2}}\int_{\mathbb{R}^N} a(x) V(x)^{r/2}|u|^{r}dx\\ \leq& \frac{1}{V_0^{r/2}}|a|_{\frac{2}{2-r}} \Big|V^{r/2}|u|^{r} \Big|_{\frac{2}{r}}\\ =& \frac{1}{V_0^{r/2}}|a|_{\frac{2}{2-r}} \|u\|^{r}\\ \end{aligned} \end{equation} and so $\varphi$ defined by \eqref{2.1} is well defined on $E$. Next, we prove that \eqref{2.2} holds. For any $\lambda\in(0,1)$, one can deduce from $H(f)(1)$ and the H\"older inequality that \begin{equation}\label{2.5} \begin{aligned} &\int_{\mathbb{R}^N}\max_{t\in[0,1]}|f(x,u+th)h|dx\\ &\leq \int_{\mathbb{R}^N}\max_{t\in[0,1]}|f(x,u+th)||h|dx\\ &\leq \int_{\mathbb{R}^N}a(x)(|u|+|h|)^{r-1}|h|dx\\ &\leq \int_{\mathbb{R}^N}ra(x)(|u|^{r-1}+|h|^{r-1})|h|dx\\ &=\int_{\mathbb{R}^N}r\frac{a(x)}{V^{r/2}}\Big[(V^{\frac{r-1}{2}}|u|^{r-1})(V^{1/2}|h|)+V^{r/2}|h|^{r}\Big]dx\\ &\leq \frac{r}{V_0^{r/2}}\int_{\mathbb{R}^N} a(x) \Big[(V^{\frac{r-1}{2}}|u|^{r-1})(V^{1/2}|h|)+V^{r/2}|h|^{r}\Big]dx\\ &\leq \frac{r}{V_0^{r/2}}\Big[|a|_{\frac{2}{2-r}}\big|V^{\frac{r-1}{2}}|u|^{r-1}\big|_{\frac{2}{r-1}}\big|V^{1/2}|h|\big|_2+|a|_{\frac{2}{2-r}} \Big|V^{r/2}|h|^{r} \Big|_{\frac{2}{r}}\Big]\\ &\leq \frac{r}{V_0^{r/2}}|a|_{\frac{2}{2-r}} \Big[\|u\|^{ r-1 }\|h\|+\|h\|^{ r }\Big]\\ &\leq \frac{r}{V_0^{r/2}}|a|_{\frac{2}{2-r}} \big[\|u\|^{ r-1 }+\|h\|^{ r-1 }\big]\|h\| <+\infty. \end{aligned} \end{equation} Thus, by \eqref{2.1}, \eqref{2.5} and Lebesgue's Dominated Convergence Theorem, we have \begin{equation}\label{2.6} \begin{aligned} \langle\varphi'(u),v\rangle &=\lim_{t\to 0^+}\frac{\varphi(u+th)-\varphi(u)}{t}\\ &= \int_{\mathbb{R}^N}\Big[ (-\Delta)^{\alpha/2}u(-\Delta)^{\alpha/2}v+V(x)uv\Big]dx\\ &\quad -\lim_{t\to0^+}\int_{\mathbb{R}^N}\frac{F(x,u+th)-F(x,u)}{t}dx\\ &= \int_{\mathbb{R}^N}\Big[ (-\Delta)^{\alpha/2}u(-\Delta)^{\alpha/2}v+V(x)uv\Big]dx\\ &\quad -\lim_{t\to 0^+}\int_{\mathbb{R}^N}f(x,u+t\lambda h)h\,dx\\ &= \int_{\mathbb{R}^N}\Big[ (-\Delta)^{\alpha/2}u(-\Delta)^{\alpha/2}v+V(x)uv\Big]dx -\lim_{t\to 0^+}\int_{\mathbb{R}^N}f(x,u)h\,dx \end{aligned} \end{equation} which implies that \eqref{2.2} holds. Moreover, by a standard argument, it is easy to show that the critical points of $\varphi$ in $E$ are solutions of problem \eqref{eP} (see \cite{15}). Next, we prove that $\varphi'$ is continuous on $E$. According to \eqref{2.1}, it suffices to show that \[ J'(u)=\int_{\mathbb{R}^N}f(x,u)dx. \] is continuous. Let $u_n\to u$ in $E$, then $u_n\to u$ in $L^2(\mathbb{R}^N)$, since the imbedding $E\hookrightarrow H^{\alpha}(\mathbb{R}^N)\hookrightarrow L^{2}(\mathbb{R}^N)$ is continuous. Thus, \begin{equation}\label{2.7} u_n(x)\to u(x),\;{\rm a.e.}\;x\in\mathbb{R}^N. \end{equation} \indent We claim that \begin{equation}\label{2.8} \lim_{n\to+\infty}\int_{\mathbb{R}^N}\big|f(x,u_n(x))-f(x,u(x))\big|^2dx=0. \end{equation} Otherwise, there exists a constant $\varepsilon>0$ and a subsequence $\{u_{n_k}\}_{k=1}^\infty$ such that \begin{equation}\label{2.9} \int_{\mathbb{R}^N}\big|f(x,u_{n_k}(x))-f(x,u(x))\big|^2dx\geq\varepsilon,\; \forall k\geq 1. \end{equation} Since $u_n\to u$ in $L^2(\mathbb{R}^N)$, passing to a subsequence if necessary, it can be assumed that \[ C=:\sum_{k=1}^\infty |u_{n_k}-u|_2^2<+\infty. \] Set $w(x)=\big(\sum_{k=1}^\infty |u_{n_k}(x)-u(x)|^2\big)^{1/2}$, $x\in \mathbb{R}^N$. Then $w\in L^2(\mathbb{R}^N)$. Therefore, \begin{align*} &\int_{\mathbb{R}^N}\big|f(x,u_{n_k}(x))-f(x,u(x))\big|^2dx\\ &\leq 2\int_{\mathbb{R}^N} \big(|f(x,u_{n_k}(x))|^2+|f(x,u(x))|^2\big)dx\\ &\leq 2\int_{\mathbb{R}^N} |a(x)|^2\big[|u_{n_k}(x)|^{2(r-1)}+|u(x)|^{2(r-1)}\big]dx\\ &= 2\int_{\mathbb{R}^N} |a(x)|^2\big[|u_{n_k}(x)-u(x)+u(x)|^{2(r-1)}+|u(x)|^{2(r-1)}\big]dx\\ &\leq 2\int_{\mathbb{R}^N} |a(x)|^2\big[(|u_{n_k}(x)-u(x)|+|u(x)|)^{2(r-1)}+|u(x)|^{2(r-1)}\big]dx\\ &\leq 24^{r-1}\int_{\mathbb{R}^N} |a(x)|^2\big[|w(x)|^{2(r-1)}+|u(x)|^{2(r-1)}+|u(x)|^{2(r-1)}\big]dx\\ &\leq 4^{r}\int_{\mathbb{R}^N} |a(x)|^2\Big[|w(x)|^{2(r-1)}+|u(x)|^{2(r-1)}\big]dx\\ &\leq 4^{r} |a^2|_{\frac{1}{2-r}}\big[\big||w(x)|^{2(r-1)}\big|_{\frac{1}{r-1}}+\big||u(x)|^{2(r-1)}\big|_{\frac{1}{r-1}}\Big]dx\\ &=4^{r} |a|_{\frac{2}{2-r}}^2\big[\big||w(x)|_2^{2(r-1)}+ |u|_2^{2(r-1)}\Big]dx\\ &\leq 4^{r} |a|_{\frac{2}{2-r}}^2\big[\big||w(x)|_2^{2(r-1)}+ |u|_2^{2(r-1)}\Big]dx <+\infty. \end{align*} Then by \eqref{2.7} and Lebesgue's Dominated Convergence Theorem, we have \[ \lim_{k\to+\infty}\int_{\mathbb{R}^N}\big|f(x,u_{n_k}(x))-f(x,u(x))\big|^2dx=0, \] which contradicts with \eqref{2.7}. Hence \eqref{2.8} holds. Applying \eqref{2.2}, \eqref{2.8} and the H\"older inequality, we have \begin{align*} &|\langle J'(u_n)-J'(u),v\rangle|\\ &=\Big|\int_{\mathbb{R}^N}(f(x,u_n(x))-f(x,u(x)))v(x)dx\Big|\\ &\leq \int_{\mathbb{R}^N}|f(x,u_n(x))-f(x,u(x))||v(x)|dx \\ &\leq \Big(\int_{\mathbb{R}^N}|f(x,u_n(x))-f(x,u(x))|^2dx\Big)^{1/2} \Big(\int_{\mathbb{R}^N}|v(x)|^2dx\Big)^{1/2}\\ &\leq \Big(\int_{\mathbb{R}^N}|f(x,u_n(x))-f(x,u(x))|^2dx\Big)^{1/2} \Big(\int_{\mathbb{R}^N}\frac{V(x)}{V_0}|v(x)|^2dx\Big)^{1/2}\\ &= \frac{1}{V_0^{1/2}}\Big(\int_{\mathbb{R}^N}|f(x,u_n(x))-f(x,u(x))|^2dx\Big)^{1/2} \Big(\int_{\mathbb{R}^N} V(x) |v(x)|^2dx\Big)^{1/2}\\ &\leq \frac{1}{V_0^{1/2}}\Big(\int_{\mathbb{R}^N}|f(x,u_n(x))-f(x,u(x))|^2dx \Big)^{1/2}\|v\|^{1/2}\\ &\to 0,\quad\text{as } n\to+\infty. \end{align*} This shows that $J'$ is continuous, and so $\varphi'$ is continuous. The proof is completed. \end{proof} \begin{lemma}[\cite{16}] \label{lem2.3} Let $X$ be a real Banach space and $\varphi \in C^1(X,\mathbb{R})$ satisfies the (PS)-condition. If $\varphi$ is bounded from blow, then $c=\inf_{u\in X} \varphi(u)$ is a critical value of $\varphi$. \end{lemma} To find multiplicity of nontrivial critical points of $\varphi$, the following ``genus'' properties are needed in our argument. Let $X$ be a Banach space, $\varphi\in C^1 (X, \mathbb{R})$ and $c\in \mathbb{R}$. Set \begin{gather*} \Sigma=\{ A\subset X\setminus \{0\}: A\text{ is closed in $X$ and symmetric with respect to} 0\},\\ K_c^{\varphi}=\{u\in X: \varphi(u)=c,\, \varphi'(u)=0\}\text{ and } \varphi^c=\{u\in X: \varphi(u)\leq c\}. \end{gather*} \begin{definition}[\cite{17}] \rm For $A \in \Sigma$, we say genus of $A$ is $n$ denoted by $\gamma(A)=n$ if there is an odd map $\phi \in C(A, \mathbb{R}^n\backslash\{0\})$ and $n$ is the smallest integer with this property. \end{definition} \begin{definition}[\cite{18}] \rm Let $X$ be a Banach space with $X^*$ being its topological dual and $\varphi\in C^1(X)$. We say that $\varphi$ satisfies the $Palais-Smale$ condition at level $c\in\mathbb{R}$ ($PS_c$-condition for short), if any sequence $\{x_n\}_{n=1}^\infty\subseteq X$, such that $$ \varphi(x_n)\to c,\;\;\varphi'(x_n)\to 0\;{\rm in}\; X^*, $$ has a strongly subsequence. If this is true at every level $c\in\mathbb{R}$, then we simply say that $\varphi$ satisfies the $Palais-Smale$ condition ($PS$-condition for short). \end{definition} The notion of genus generalizes the concept of dimension of a linear space. \begin{lemma}[{\cite[Proposition 4.2.15]{18}}] \label{lem2.4} If $X$ is a Banach space and $U$ is a bounded symmetric neighborhood of the origin in $X$, then $\gamma(\partial U) = {\rm dim} X$. \end{lemma} \begin{lemma}[\cite{17}] \label{lem2.5} Let $\varphi$ be an even $C^1$ functional on $X$ and satisfy the (PS)-condition. For any $n\in N$, set $$ \Sigma_n=\{A\in\Sigma: \gamma(A)\geq n\}\;{\rm and}\; c_n=\inf_{A\in\Sigma_n}\sup_{u\in A}\varphi(u). $$ \begin{itemize} \item[(a)] If $\Sigma_n\neq\emptyset $ and $c_n\in\mathbb{R}$, then $c_n$ is a critical value of $\varphi$; \item[(b)] If There exists $l\in N$ such that $c_l=c_{l+1}=\dots=c_{l+n}=c<+\infty$, then $\gamma(K_c^{\varphi})\geq n+1$. \end{itemize} \end{lemma} \section{Proofs of main results} \begin{proof}[Proof of Theorem \ref{thm1.1}] We first prove that $\varphi$ is bounded from below. By (A3)(1), one yields \begin{equation}\label{3.1} \begin{aligned} |F(x,t)|=&\Big|F(x,0)+\int_0^t\frac{d}{ds}F(x,s)ds\Big|\\ =& \Big|\int_0^t f(x,s)ds\Big|\\ \leq&\int_0^t | f(x,s) | ds\\ \leq& \frac{a(x)}{r}|t|^{r} \leq a(x) |t|^{r}, \end{aligned}\end{equation} for all $x\in \mathbb{R}^N$ and all $t\in\mathbb{R}$. Hence, from \eqref{2.4} and \eqref{3.1}, we obtain \begin{equation}\label{3.2} \begin{aligned} \varphi(u)=&\frac{1}{2}\int_{\mathbb{R}^N}\Big[|(-\Delta)^{\alpha/2}u(x)|^{2}+V(x)u^2\Big]dx-\int_{\mathbb{R}^N}F(x,u)dx\\ \geq& \frac{1}{2}\|u\|^{2}-\int_{\mathbb{R}^N}a(x)|u|^{r}dx\\ \geq& \frac{1}{2}\|u\|^{2}- \frac{1}{V_0^{r/2}}|a|_{\frac{2}{2-r}} \|u\|^{ r }. \end{aligned} \end{equation} Since $10$ such that \begin{equation}\label{3.3} |u_n|_2\leq C_0\|u_n\|\leq C_1,\quad \forall n\in N. \end{equation} So we may assume, going if necessary to a subsequence, that $$ u_n\rightharpoonup u_0\quad\text{weakly in } E. $$ From the choice of the function $a\in L^{\frac{2}{2-r}}(\mathbb{R}^N)$, for any given number $\varepsilon>0$, we can choose $R_\varepsilon>0$ such that \begin{equation}\label{3.5} \Big(\int_{|x|>R_\varepsilon} |a(x)|^{\frac{2}{2-r}} dx\Big)^{\frac{2-r}{2}}<\varepsilon. \end{equation} Since the embedding $E\hookrightarrow L_{\rm loc}^{2}(\mathbb{R}^N)$ is compact, $u_n\rightharpoonup u_0$ in $E$ implies $u_n\to u_0$ in $L_{\rm loc}^{2}(\mathbb{R}^N)$, and hence, \begin{equation}\label{3.6} \lim_{n\to+\infty} \int_{|x|\leq R_\varepsilon} |u_n-u_0|^{2} dx=0. \end{equation} Let $B_\varepsilon=\{x\in\mathbb{R}^N:|x|\leq R_\varepsilon\}$ and $B_\varepsilon^c=\mathbb{R}^N\setminus B_\varepsilon$. By \eqref{3.6}, there exists $n_0\in N$ such that \begin{equation}\label{3.7} |u_n-u_0|_{L^{2}(B_\varepsilon)}<\varepsilon,\;{\rm for}\;n\geq n_0. \end{equation} Next, we prove that $$ \int_{\mathbb{R}^N}[f(x,u_n)-f(x,u_0)](u_n-u_0)dx\to 0,\quad\text{as } n\to+\infty. $$ Indeed, by hypothesis (A3)(1), we have \begin{equation}\label{3.10} \begin{aligned} &\int_{\mathbb{R}^N}|f(x,u_n)-f(x,u_0)|| u_n-u_0|dx\\ &\leq \int_{\mathbb{R}^N}a(x)[|u_n|^{r-1}+|u_0|^{r-1}]| u_n-u_0|dx\\ &=\int_{\mathbb{R}^N}\frac{a(x)}{V^{\frac{r-1}{2}}} V^{\frac{r-1}{2}}[|u_n|^{r-1}+|u_0|^{r-1}]| u_n-u_0|dx\\ &\leq {V_0}^{-\frac{r-1}{2}} \int_{\mathbb{R}^N} a(x) V^{\frac{r-1}{2}}[|u_n|^{r-1}+|u_0|^{r-1}]| u_n-u_0|dx\\ &\leq {V_0}^{-\frac{r-1}{2}}\Big[ \int_{B_\varepsilon} a(x) V^{\frac{r-1}{2}} [|u_n|^{r-1}+|u_0|^{r-1}]| u_n-u_0|dx\\ &\quad +\int_{B_\varepsilon^c} a(x) V^{\frac{r-1}{2}}[|u_n|^{r-1}+|u_0|^{r-1}]| u_n-u_0|dx\Big]\\ &=: {V_0}^{-\frac{r-1}{2}}[I_1+I_2]. \end{aligned}\end{equation} On the one hand, using the H\"older inequality and \eqref{3.7}, we have \begin{equation}\label{3.11} \begin{aligned} I_1 &= \int_{B_\varepsilon} a(x) V^{\frac{r-1}{2}}[|u_n|^{r-1}+|u_0|^{r-1}]| u_n-u_0|dx\\ &\leq |a|_{L^{\frac{2}{2-r}}(B_\varepsilon)}\Big[\big|V^{\frac{r-1}{2}}|u_n|^{r-1} \big|_{L^{\frac{2}{r-1}}(B_\varepsilon)} \\ &\quad +\big|V^{\frac{r-1}{2}}|u_0|^{r-1} \big|_{L^{\frac{2}{r-1}}(B_\varepsilon)}\Big]|u_n-u_0|_{L^{2}(B_\varepsilon)}\\ &\leq \varepsilon|a|_{L^{\frac{2}{2-r}}(B_\varepsilon)} \Big[\big|V^{\frac{r-1}{2}}|u_n|^{r-1} \big|_{L^{\frac{2}{r-1}}(B_\varepsilon)} +\big|V^{\frac{r-1}{2}}|u_0|^{r-1} \big|_{L^{\frac{2}{r-1}}(B_\varepsilon)}\Big]\\ &\leq \varepsilon|a|_{L^{\frac{2}{2-r}}(\mathbb{R}^N)} \Big[\big|V^{\frac{r-1}{2}}|u_n|^{r-1} \big|_{L^{\frac{2}{r-1}}(\mathbb{R}^N))} +\big|V^{\frac{r-1}{2}}|u_0|^{r-1} \big|_{L^{\frac{2}{r-1}}(\mathbb{R}^N))}\Big]\\ &= \varepsilon|a|_{L^{\frac{2}{2-r}}(\mathbb{R}^N)}[\|u_n\|^{r-1}+\|u_0\|^{r-1}]\\ &\leq \varepsilon|a|_{L^{\frac{2}{2-r}}(\mathbb{R}^N)} \big[\big(\frac{C_1}{C_0}\big)^{r-1}+\|u_0\|^{r-1}\big] \end{aligned} \end{equation} for all $n\geq n_0$. On the other hand, using the H\"older inequality and \eqref{3.5}, we have \begin{equation}\label{3.12} \begin{aligned} I_2&= \int_{B_\varepsilon^c} a(x) V^{\frac{r-1}{2}}[|u_n|^{r-1}|+|u_0|^{r-1}]| u_n-u_0|dx\\ &\leq |a|_{L^{\frac{2}{2-r}}(B_\varepsilon^c)}\Big[\big|V^{\frac{r-1}{2}}|u_n|^{r-1} \big|_{L^{\frac{2}{r-1}}(B_\varepsilon^c)}\\ &\quad +\big|V^{\frac{r-1}{2}}|u_0|^{r-1} \big|_{L^{\frac{2}{r-1}}(B_\varepsilon^c)}\Big]|u_n-u_0|_{L^{2}(B_\varepsilon^c)}\\ &\leq |a|_{L^{\frac{2}{2-r}}(B_\varepsilon^c)}\Big[\big|V^{\frac{r-1}{2}}|u_n|^{r-1} \big|_{L^{\frac{2}{r-1}}(\mathbb{R}^N)}\\&+\big|V^{\frac{r-1}{2}}|u_0|^{r-1} \big|_{L^{\frac{2}{r-1}}(\mathbb{R}^N)}\Big]|u_n-u_0|_{L^{2}(\mathbb{R}^N)}\\ &\leq C_0\varepsilon \|u_n-u_0\|\Big[\big|V^{\frac{r-1}{2}}|u_n|^{r-1} \big|_{L^{\frac{2}{r-1}}(\mathbb{R}^N)} +\big|V^{\frac{r-1}{2}}|u_0|^{r-1} \big|_{L^{\frac{2}{r-1}}(\mathbb{R}^N)}\Big]\\ &\leq C_0\varepsilon \|u_n-u_0\|\big[\|u_n\|^{r-1}+\|u_0\|^{r-1}\big]\\ &\leq 2C_0\varepsilon \big[\|u_n\|^{r}+\|u_0\|^{r}\big]\\ &\leq 2C_0\varepsilon \Big[\big(\frac{C_1}{C_0}\big)^{r}+\|u_0\|^{r}\Big]\\ \end{aligned} \end{equation} for all $n\in N$. Since $\varepsilon$ is arbitrary, it follows from \eqref{3.10}, \eqref{3.11} and \eqref{3.12} that \begin{equation}\label{3.13} \int_{\mathbb{R}^N}[f(x,u_n)-f(x,u_0)](u_n-u_0)dx\to 0\quad\text{as } n\to+\infty. \end{equation} In view of the definition of weak convergence, we have \begin{equation}\label{3.14} \langle \varphi'(u_n)-\varphi'(u_0),u_n-u_0\rangle\to 0,\quad\text{as }n\to+\infty. \end{equation} Note that \begin{equation}\label{3.15} \begin{aligned} &\langle \varphi'(u_n)-\varphi'(u_0),u_n-u_0\rangle\\ &= \int_{\mathbb{R}^N}\Big[ \big|(-\Delta)^{\alpha/2}(u_n-u_0)\big|^2+V(x)(u_n-u_0)^2\Big]dx\\ &\quad -\int_{\mathbb{R}^N}[f(x,u_n)-f(x,u_0)](u_n-u_0)dx\\ &=\|u_n-u_0\|^2-\int_{\mathbb{R}^N}[f(x,u_n)-f(x,u_0)](u_n-u_0)dx. \end{aligned} \end{equation} From \eqref{3.13}, \eqref{3.14} and \eqref{3.15} it follows that \begin{equation}\label{3.16} \|u_n-u_0\|\to 0,\quad\text{as }n\to+\infty, \end{equation} which implies that $u_n\to u_0$ in $E$. Therefore $\varphi$ satisfies the $(PS)$-condition. Then by Lemma \ref{lem2.3} we see that $c=\inf_{u\in E}\varphi(u)$ is a critical value of $\varphi$, i.e., there exists a critical point $u_0\in E$ such that $\varphi(u_0)=c$. Finally, we prove that $u_0\neq 0$. Taking $\phi\in[H_0^{\alpha}(I)\cap E]\setminus\{0\}$ with $\|\phi\|=1$, then by (A3)(2), for $t\in(0,1)$, we have \begin{equation}\label{3.17} \begin{aligned} \varphi(t\phi)=&\frac{1}{2}\int_{\mathbb{R}^N}\Big[|(-\Delta)^{\alpha/2}(t\phi)\big|^2+V(x)| t\phi|^{2}\Big]dx- \int_{\mathbb{R}^N}F(x,t\phi)dx\\ =& \frac{1}{2}t^{2}\|\phi\|^{2}-\int_{I}F(x,t\phi)dx\\ <& \frac{1}{2}t^{2}-\int_{I}\rho|t\phi|^{\theta}dx\\ =& \frac{1}{2}t^{2}-t^{\theta}\rho\int_{I}|\phi|^{\theta}dx. \end{aligned} \end{equation} Since $1<\theta<2$, it follows \eqref{3.17} that $\varphi(t\phi)<0$ for $t>0$ small enough. Hence $c=\varphi(u_0)<0$. Therefore $u_0$ is a nontrivial critical point of $\varphi$ with $\varphi(u_0)=\inf_{u\in E}\varphi(u)$ and is a nontrivial solution of problem \eqref{eP}. The proof is completed. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.2}] From the proof of Theorem \ref{thm1.1}, we know that $\varphi$ is bounded below and satisfies the $(PS)$-condition. It is clear from $F(x,-t)=F(x,t)$ that $\varphi$ is even and $\varphi(0)=0$. In order to apply Lemma \ref{lem2.5}, we prove now that for any $n\in N$, there exists $K\subseteq H^{\alpha}(\mathbb{R}^N)$ compact, and symmetric with $\gamma(K)=n$ such that $$\sup_{u\in K}\varphi(u)<0.$$ For any $n\in N$, we take $n$ disjoint open sets $I_i$ such that $\cup_{i=1}^n I_i\subset I$. For $i=1,2,\dots,n$, we choose $u_i\in (H_0^{\alpha}(I_i)\cap E)\setminus\{0\}$ and $|u_i|_{\theta}=1$. Let $E_n={\rm span}\{u_1,u_2,\dots,u_m\}$. Because $E_n$ is a finite dimensional subspace of $E$, all norm are equivalent and so we can find $0