\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2016 (2016), No. 40, pp. 1--13.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2016 Texas State University.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2016/40\hfil Fractional Hamiltonian systems] {Existence of solutions to fractional Hamiltonian systems with combined nonlinearities} \author[Z. Zhang, R. Yuan \hfil EJDE-2016/40\hfilneg] {Ziheng Zhang, Rong Yuan} \address{Ziheng Zhang \newline Department of Mathematics, Tianjin Polytechnic University, Tianjin 300387, China} \email{zhzh@mail.bnu.edu.cn} \address{Rong Yuan \newline Department of Mathematical Sciences, Beijing Normal University, Beijing 100875, China} \email{ryuan@bnu.edu.cn} \thanks{Submitted November 11, 2015. Published January 27, 2016.} \subjclass[2010]{34C37, 35A15, 35B38} \keywords{Fractional Hamiltonian systems; critical point; \hfill\break\indent variational methods; mountain pass theorem} \begin{abstract} This article concerns the existence of solutions for the fractional Hamiltonian system \begin{gather*} - _tD^{\alpha}_{\infty}\big(_{-\infty}D^{\alpha}_{t}u(t)\big) -L(t)u(t)+\nabla W(t,u(t))=0,\\ u\in H^{\alpha}(\mathbb{R},\mathbb{R}^n), \end{gather*} where $\alpha\in (1/2,1)$, $L\in C(\mathbb{R},\mathbb{R}^{n^2})$ is a symmetric and positive definite matrix. The novelty of this article is that if $\tau_1 |u|^2\leq (L(t)u,u)\leq \tau_2 |u|^2$ and the nonlinearity $W(t,u)$ involves a combination of superquadratic and subquadratic terms, the Hamiltonian system possesses at least two nontrivial solutions. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{remark}[theorem]{Remark} \allowdisplaybreaks \newcommand{\abs}[1]{\lvert#1\rvert} \section{Introduction} The study of fractional calculus (differentiation and integration of arbitrary order) has emerged as an important and popular field of research. It is mainly because of the extensive application of fractional differential equations in many engineering and scientific disciplines such as physics, chemistry, biology, economics, control theory, signal and image processing, biophysics, blood flow phenomena, aerodynamics, fitting of experimental data, etc., \cite{ATMS04,Hilfer00,KST06,MR93,Pod99,Zaslavsky05}. An important characteristic of a fractional-order differential operator that distinguishes it from an integer-order differential operator is its nonlocal behavior, that is, the future state of a dynamical system or process involving fractional derivatives depends on its current state as well its past states. In other words, differential equations of arbitrary order describe memory and hereditary properties of various materials and processes. This is one of the features that has contributed to the popularity of the subject and has motivated researchers to focus on fractional order models, which are more realistic and practical than the classical integer-order models. Recently, also equations including both left and right fractional derivatives are discussed. Apart from their possible applications, equations with left and right derivatives is an interesting and new field in fractional differential equations theory. In this topic, many results are obtained dealing with the existence and multiplicity of solutions of nonlinear fractional differential equations by using techniques of nonlinear analysis, such as fixed point theory (including Leray-Schauder nonlinear alternative) \cite{BaiLu05}, topological degree theory (including co-incidence degree theory) \cite{Jiang11} and comparison method (including upper and lower solutions and monotone iterative method) \cite{Zhang11} and so on. It should be noted that critical point theory and variational methods have also turned out to be very effective tools in determining the existence of solutions for integer order differential equations. The idea behind them is trying to find solutions of a given boundary value problem by looking for critical points of a suitable energy functional defined on an appropriate function space. In the last 30 years, the critical point theory has become a wonderful tool in studying the existence of solutions of differential equations with variational structures, we refer the reader to the books due to Mawhin and Willem \cite{MawhinW89}, Rabinowitz \cite{Rab86}, Schechter \cite{Schechter99} and the references listed therein. Motivated by the above classical works, in the recent paper \cite{JZ12}, the authors showed that the critical point theory is an effective approach to tackle the existence of solutions for the fractional boundary value problem \begin{gather*} _tD^{\alpha}_{T}(_0D^{\alpha}_{t}u(t))=\nabla W(t,u(t)), \quad \text{a.e. } t\in [0,T],\\ u(0)=u(T), \end{gather*} where $\alpha\in (1/2,1)$, $u\in \mathbb{R}^n$, $W\in C^1([0,T]\times\mathbb{R}^n,\mathbb{R})$ and $\nabla W(t,u)$ is the gradient of $W(t,u)$ at $u$ and obtained the existence of at least one nontrivial solution. Inspired by this work, Torres \cite{Torres12} considered the fractional Hamiltonian system \begin{equation} \begin{gathered} _tD^{\alpha}_{\infty}(_{-\infty}D^{\alpha}_{t}u(t))+L(t)u(t)=\nabla W(t,u(t)),\\ u\in H^{\alpha}(\mathbb{R},\mathbb{R}^n), \end{gathered} \label{FHS} \end{equation} where $\alpha\in (1/2,1)$, $t\in \mathbb{R}$, $u\in \mathbb{R}^n$, $L\in C(\mathbb{R},\mathbb{R}^{n^2})$ is a symmetric and positive definite matrix for all $t\in \mathbb{R}$, $W\in C^1(\mathbb{R}\times\mathbb{R}^n,\mathbb{R})$ and $\nabla W(t,u)$ is the gradient of $W(t,u)$ at $u$. Assuming that $L(t)$ and $W(t,u)$ satisfy the following hypotheses, Torres \cite{Torres12} showed that \eqref{FHS} possesses at least one nontrivial solution, using the mountain pass theorem and the following assumptions: \begin{itemize} \item[(A1)] $L(t)$ is a positive definite symmetric matrix for all $t\in \mathbb{R}$ and there exists an $l\in C(\mathbb{R},(0,\infty))$ such that $l(t)\to \infty$ as $|t|\to \infty$ and \begin{equation}\label{eqn:L coercive} (L(t)u,u)\geq l(t)|u|^2 \quad \text{for all } t\in \mathbb{R},\; u\in \mathbb{R}^n; \end{equation} \item[(A2)] $W\in C^1(\mathbb{R}\times \mathbb{R}^n,\mathbb{R})$ and there is a constant $\theta>2$ such that $$ 0<\theta W(t,u)\leq (\nabla W(t,u),u)\quad \text{for all } t\in \mathbb{R} ,\; u\in \mathbb{R}^n\backslash\{0\}; $$ \item[(A3)] $|\nabla W(t,u)|=o(|u|)$ as $|u|\to 0$ uniformly with respect to $t\in \mathbb{R}$; \item[(A4)] there exists $\overline{W}\in C(\mathbb{R}^n,\mathbb{R})$ such that $$ |W(t,u)|+|\nabla W(t,u)|\leq |\overline W(u)|\quad \text{for every } t\in \mathbb{R},\; u\in \mathbb{R}^n. $$ \end{itemize} A strong motivation for investigating \eqref{FHS} comes from fractional advection-dispersion equation (ADE for short). This is a generalization of the classical ADE in which the second-order derivative is replaced with a fractional-order derivative. In contrast to the classical ADE, the fractional ADE has solutions that resemble the highly skewed and heavy-tailed breakthrough curves observed in field and laboratory studies (see, \cite{BensonSMW01,BensonWM001,BensonWM002}), in particular in contaminant transport of ground-water flow (see, \cite{BensonWM002}). Benson et al. stated that solutes moving through a highly heterogeneous aquifer violations violates the basic assumptions of local second-order theories because of large deviations from the stochastic process of Brownian motion. In \eqref{FHS}, if $\alpha=1$, then it reduces to the following second order Hamiltonian system \begin{equation} \ddot u-L(t) u+\nabla W(t,u)=0.\label{HS} \end{equation} It is well known that the existence of homoclinic solutions for Hamiltonian systems and their importance in the study of the behavior of dynamical systems have been recognized from Poincar\'{e} \cite{Poincare}. They may be ``organizing centers" for the dynamics in their neighborhood. From their existence one may, under certain conditions, infer the existence of chaos nearby or the bifurcation behavior of periodic orbits. During the past two decades, with the works of \cite{Omana92} and \cite{Rab90} variational methods and critical point theory have been successfully applied for the search of the existence and multiplicity of homoclinic solutions of \eqref{HS}. Assuming that $L(t)$ and $W(t,u)$ are independent of $t$ or periodic in $t$, many authors have studied the existence of homoclinic solutions for \eqref{HS}, see for instance \cite{Co91,Ding95,Rab90} and the references therein and some more general Hamiltonian systems are considered in recent papers \cite{Izydorek05,Izydorek07}. In this case, the existence of homoclinic solutions can be obtained by going to the limit of periodic solutions of approximating problems. If $L(t)$ and $W(t,u)$ are neither autonomous nor periodic in $t$, the existence of homoclinic solutions of \eqref{HS} is quite different from the periodic systems, because of the lack of compactness of the Sobolev embedding, such as \cite{Ding95,Omana92,Rab91} and the references mentioned there. Assumption (A2) is the so-called global Ambrosetti-Rabinowitz condition, which implies that $W(t,u)$ is of superquadratic growth as $|u|\to \infty$. Motivated by \cite{Torres12}, in \cite{ChenHeTang15} the authors gave some more general superquadratic conditions on $W(t,u)$ and obtained that \eqref{FHS} possesses infinitely many nontrivial solutions. Furthermore, using the genus properties of critical point theory, in \cite{ZhangYuan} the authors established some new criterion to guarantee the existence of infinitely many solutions of \eqref{FHS} for the case that $W(t,u)$ is subquadratic as $|u|\to \infty$. In \cite{ChenHeTang15,ZhangYuan}, the condition (A1) is needed to guarantee that the functional corresponding to \eqref{FHS} satisfies the (PS) condition. As is well-known that condition (A1) is the so-called coercive condition and is very restrictive. In fact, for a simple choice like $L(t)=\tau Id_n$, the condition \eqref{eqn:L coercive} is not satisfied, where $\tau>0$ and $Id_n$ is the $n\times n$ identity matrix. Therefore, in \cite{ZhangYuan14} the authors focused their attention on the case that $L(t)$ is bounded in the sense that \begin{itemize} \item[(A1')] $L\in C(\mathbb{R},\mathbb{R}^{n^2})$ is a symmetric and positive definite matrix for all $t\in \mathbb{R}$ and there are constants $0<\tau_1<\tau_2<\infty$ such that $$ \tau_1|u|^2\leq (L(t)u,u)\leq \tau_2|u|^2\quad \text{for all } (t,u)\in \mathbb{R}\times \mathbb{R}^n. $$ \end{itemize} If the potential $W(t,u)$ is assumed to be subquadratic as $|u|\to \infty$, then they showed that \eqref{FHS} possessed infinitely many solutions. More recently, the authors in \cite{Torres14} and \cite{WuZhang} investigated the perturbed fractional Hamiltonian system \begin{equation} \begin{gathered} - _tD^{\alpha}_{\infty}(_{-\infty}D^{\alpha}_{t}u(t))-L(t)u(t)+\nabla W(t,u(t)) =f(t),\\ u\in H^{\alpha}(\mathbb{R},\mathbb{R}^n), \end{gathered} \label{PFHS} \end{equation} where $\alpha\in (1/2,1)$, $t\in \mathbb{R}$, $u\in \mathbb{R}^n$, $L\in C(\mathbb{R},\mathbb{R}^{n^2})$ is a symmetric and positive definite matrix for all $t\in \mathbb{R}$, $W\in C^1(\mathbb{R}\times\mathbb{R}^n,\mathbb{R})$ and $\nabla W(t,u)$ is the gradient of $W(t,u)$ at $u$, $f\in C(\mathbb{R},\mathbb{R}^n)$ and belongs to $L^2(\mathbb{R},\mathbb{R}^n)$. Some reasonable assumptions on $L$, $W(t,u)$ and $f$ are established to guarantee the existence of solutions of \eqref{PFHS}. For the other works related to fractional Hamiltonian systems, we refer the reader to \cite{NyamoradiZhou14,Torres15}. Motivated mainly by \cite{ChenHeTang15,Torres12,ZhangYuan14}, in this article we investigate the case that the nonlinearity $W(t,u)$ involves a combination of superquadratic and subquadratic terms. That is, $W(t,u)$ is of the form $$ W(t,u)=W_1(t,u)+W_2(t,u), $$ where $W_1(t,u)$ is superquadratic as $|u|\to \infty$ and $W_2(t,u)$ is of subquadratic growth at infinity. As far as the authors know, there is no literature to consider the combined nonlinearity associated with \eqref{FHS}. Therefore, we focus our our attention on this problem and provide some reasonable assumptions on $W_1(t,u)$ and $W_2(t,u)$ to guarantee the existence of at least two nontrivial solutions of \eqref{FHS}. For the statement of our main result, $W(t,u)$ is supposed to satisfy the following hypothesis: \begin{itemize} \item[(A5)] $W_1\in C^1(\mathbb{R}\times \mathbb{R}^n,\mathbb{R})$ and there exists a constant $\theta>2$ such that $$ 0<\theta W_1(t,u)\leq (\nabla W_1(t,u),u) \quad \text{for all } t\in \mathbb{R},\; u\in \mathbb{R}^n\setminus \{0\}; $$ \item[(A6)] there exists a continuous function $a:\mathbb{R}\to \mathbb{R}^+$ with $\lim_{|t|\to \infty}a(t)=0$ such that $$ |\nabla W_1(t,u)|\leq a(t)|u|^{\theta-1} \quad \text{for all } (t,u)\in \mathbb{R}\times \mathbb{R}^n; $$ \item[(A7)] $W_2(t,0)=0$ for $t\in \mathbb{R}$, $W_2\in C^1(\mathbb{R}\times\mathbb{R}^n,\mathbb{R})$ and there exist a constant $1<\varrho<2$ and a continuous function $b:\mathbb{R}\to\mathbb{R}^+$ such that $$ b(t)|u|^\varrho \leq W_2(t,u)\quad \text{for all } (t,u)\in \mathbb{R}\times \mathbb{R}^n; $$ \item[(A8)] for all $t\in \mathbb{R}$ and $u\in \mathbb{R}^n$, $$ |\nabla W_2(t,u)|\leq c(t)|u|^{\varrho-1}, $$ where $c:\mathbb{R}\to \mathbb{R}^+$ is a continuous function such that $c\in L^\xi(\mathbb{R},\mathbb{R})$ for some constant $1\leq \xi\leq 2$. \end{itemize} To obtain the existence of at least two nontrivial solutions of \eqref{FHS}, we also need the following assumption on $a$ and $c$: \begin{itemize} \item[(A9)] $$ \Bigl(\frac{2\|c\|_{L^\xi} C_{\varrho\xi^*}^\varrho}{\varrho} \frac{\theta-\varrho}{\theta-2}\Bigr)^{\theta-2} <\Bigl(\frac{\theta}{2\|a\|_\infty C_\theta^\theta} \frac{2-\varrho}{\theta-\varrho}\Bigr)^{\varrho-2}, $$ where $\|c\|_{L^\xi}$ is the $L^\xi(\mathbb{R},\mathbb{R})$ norm of $c$, $\|a\|_\infty=\operatorname{ess\,sup}_{t\in \mathbb{R}} a(t)$, $\varrho$ and $\theta$ are defined in (A5) and (A7), respectively, $\xi^*$ is the conjugate component of $\xi$, that is, $\frac{1}{\xi}+\frac{1}{\xi^*}=1$, $C_{\varrho \xi^*}$ and $C_\theta$ are defined in \eqref{eqn:LpContXalpha} below. \end{itemize} Now, we are in a position to state our main result. \begin{theorem}\label{Thm:MainTheorem1} Suppose that {\rm (A5)-(A9)} are satisfied, then \eqref{FHS} possesses at least two nontrivial solutions. \end{theorem} \begin{remark} \label{rmk1.2} \rm In view of (A5), we deduce that (see \cite[Fact 2.1]{Izydorek05}) \begin{equation}\label{eqn:W1less1} W_1(t,u)\leq W_1(t,\frac{u}{|u|})|u|^{\theta}\quad\text{for } t\in \mathbb{R} \text{ and } 0<|u|\leq 1 \end{equation} and \begin{equation}\label{eqn:W1big1} W_1(t,u)\geq W_1(t,\frac{u}{|u|})|u|^{\theta}\quad \text{for } t\in \mathbb{R} \text{ and } |u|\geq 1. \end{equation} Moreover, by (A7) and (A8), it is obvious that \begin{equation}\label{eqn:W2Estn} W_2(t,u)\leq \frac{c(t)}{\varrho}\quad \text{for all }t\in\mathbb{R},\; u\in \mathbb{R}^n. \end{equation} In addition, from (A5)--(A8), it is easy to show that \begin{equation}\label{eqn:W12Estn} W(t,u)=\int_0^1(\nabla W(t,su),u)ds \leq \frac{a(t)}{\theta}|u|^\theta+\frac{c(t)}{\varrho}|u|^\varrho \end{equation} for all $(t,u)\in \mathbb{R}\times \mathbb{R}^n$. For the reader's convenience, we present one example to illustrate our main result. Considering the following nonlinearity: $$ W(t,u)=a(t)|u|^3+c(t)|u|^\frac{3}{2}, $$ where $a:\mathbb{R}\to \mathbb{R}^+$ and $c:\mathbb{R}\to\mathbb{R}^+$ are continuous functions, $\lim_{|t|\to \infty}a(t)=0$ and $c\in L^\xi(\mathbb{R},\mathbb{R})$ with $1\leq \xi\leq 2$. Then it is easy to check that (A5)--(A8) are satisfied. Meanwhile, the additional assumption $$ 2\|c\|_{L^\xi}C_{\varrho\xi^*}^\varrho\sqrt{2\|a\|_\infty C_\theta^\theta}<1 $$ guarantees that (A9) holds, where $\theta=3$ and $\varrho=3/2$. Here, we must point out that, in our Theorem \ref{Thm:MainTheorem1}, for the first time we obtain that \eqref{FHS} has at least two nontrivial solutions for the case that $W(t,u)$ is a combined nonlinearity. Therefore, the previous results in \cite{ChenHeTang15,Torres12,ZhangYuan14} are generalized and improved significantly. However, we do not know whether \eqref{FHS} also possesses infinitely solutions if the potential $W(t,u)$ is even with respect to $u$ as usual. \end{remark} The remaining part of this paper is organized as follows. Some preliminary results are presented in Section 2. Section 3 is devoted to the proof of Theorem \ref{Thm:MainTheorem1}. \section{Preliminary Results} In this section, for the reader's convenience, firstly we introduce some basic definitions of fractional calculus. The Liouville-Weyl fractional integrals of order $0<\alpha<1$ are defined as \begin{gather*} _{-\infty}I^{\alpha}_x u(x)=\frac{1}{\Gamma(\alpha)} \int^x_{-\infty} (x-\xi)^{\alpha-1}u(\xi)d\xi, \\ _{x}I^{\alpha}_{\infty} u(x)=\frac{1}{\Gamma(\alpha)} \int^{\infty}_{x}(\xi-x)^{\alpha-1}u(\xi)d\xi. \end{gather*} The Liouville-Weyl fractional derivative of order $0<\alpha<1$ are defined as the left-inverse operators of the corresponding Liouville-Weyl fractional integrals \begin{gather}\label{eqn:RD} _{-\infty}D^{\alpha}_x u(x)=\frac{d}{dx} {_{-\infty}I^{1-\alpha}_x u(x)},\\ \label{eqn:LD} _{x}D^{\alpha}_{\infty} u(x)=-\frac{d}{dx} {_{x}I^{1-\alpha}_{\infty} u(x)}. \end{gather} The definitions of \eqref{eqn:RD} and \eqref{eqn:LD} may be written in an alternative form as follows: \begin{gather*} _{-\infty}D^{\alpha}_x u(x)=\frac{\alpha}{\Gamma(1-\alpha)} \int^{\infty}_0 \frac{u(x)-u(x-\xi)}{\xi^{\alpha+1}}d\xi, \\ _{x}D^{\alpha}_{\infty} u(x)=\frac{\alpha}{\Gamma(1-\alpha)} \int^{\infty}_0 \frac{u(x)-u(x+\xi)}{\xi^{\alpha+1}}d\xi. \end{gather*} Moreover, recall that the Fourier transform $\widehat{u}(w)$ of $u(x)$ is defined by $$ \widehat{u}(w)=\int_{-\infty}^{\infty}e^{-iwx}u(x)dx. $$ To establish the variational structure which enables us to reduce the existence of solutions of \eqref{FHS} to find critical points of the corresponding functional, it is necessary to construct appropriate function spaces. In what follows, we introduce some fractional spaces, for more details see \cite{ErvinR06,Torres12}. To this end, denote by $L^p(\mathbb{R},\mathbb{R}^n)$ ($2\leq p <\infty$) the Banach spaces of functions on $\mathbb{R}$ with values in $\mathbb{R}^n$ under the norms $$ \|u\|_{L^p}=\Bigl(\int_{\mathbb{R}}|u(t)|^p dt\Bigr)^{1/p}, $$ and $L^{\infty}(\mathbb{R},\mathbb{R}^n)$ is the Banach space of essentially bounded functions from $\mathbb{R}$ into $\mathbb{R}^n$ equipped with the norm $$ \|u\|_{\infty}=\operatorname{ess\,sup}\{|u(t)|: t\in \mathbb{R} \}. $$ For $\alpha>0$, define the semi-norm $$ |u|_{I^{\alpha}_{-\infty}}=\|_{-\infty}D^{\alpha}_x u\|_{L^2} $$ and the norm \begin{equation}\label{eqn:defnRnorm} \|u\|_{I^{\alpha}_{-\infty}} =\Bigl(\|u\|^2_{L^2}+|u|^2_{I^{\alpha}_{-\infty}}\Bigr)^{1/2} \end{equation} and let $$ I^{\alpha}_{-\infty} =\overline{C^{\infty}_0(\mathbb{R},\mathbb{R}^n)}^{\|\cdot\|_{I^{\alpha}_{-\infty}}}, $$ where $C_0^{\infty}(\mathbb{R},\mathbb{R}^n)$ denotes the space of infinitely differentiable functions from $\mathbb{R}$ into $\mathbb{R}^n$ with vanishing property at infinity. Now we can define the fractional Sobolev space $H^{\alpha}(\mathbb{R},\mathbb{R}^n)$ in terms of the Fourier transform. Choose $0<\alpha<1$, define the semi-norm $$ |u|_{\alpha}=\||w|^{\alpha}\widehat{u}\|_{L^2} $$ and the norm $$ \|u\|_{\alpha}=\Bigl(\|u\|^2_{L^2}+|u|^2_{\alpha}\Bigr)^{1/2} $$ and let $$ H^{\alpha}=\overline{C^{\infty}_0(\mathbb{R},\mathbb{R}^n)}^{\|\cdot\|_{\alpha}}. $$ Moreover, we note that a function $u\in L^2(\mathbb{R},\mathbb{R}^n)$ belongs to $I^{\alpha}_{-\infty}$ if and only if $$ |w|^{\alpha}\widehat{u}\in L^2(\mathbb{R},\mathbb{R}^n). $$ Especially, we have $$ |u|_{I^{\alpha}_{-\infty}}=\||w|\widehat{u}\|_{L^2}. $$ Therefore, $I^{\alpha}_{-\infty}$ and $H^{\alpha}$ are equivalent with equivalent semi-norm and norm. Analogous to $I^{\alpha}_{-\infty}$, we introduce $I^{\alpha}_{\infty}$. Define the semi-norm $$ |u|_{I^{\alpha}_{\infty}}=\|_{x}D^{\alpha}_{\infty} u\|_{L^2} $$ and the norm \begin{equation}\label{eqn:defnRnormb} \|u\|_{I^{\alpha}_{\infty}}=\Bigl(\|u\|^2_{L^2} +|u|^2_{I^{\alpha}_{\infty}}\Bigr)^{1/2} \end{equation} and let $$ I^{\alpha}_{\infty}=\overline{C^{\infty}_0(\mathbb{R}, \mathbb{R}^n)}^{\|\cdot\|_{I^{\alpha}_{\infty}}}. $$ Then $I^{\alpha}_{-\infty}$ and $I^{\alpha}_{\infty}$ are equivalent with equivalent semi-norm and norm, see \cite{ErvinR06}. Let $C(\mathbb{R},\mathbb{R}^n)$ denote the space of continuous functions from $\mathbb{R}$ into $\mathbb{R}^n$. Then we obtain the following lemma. \begin{lemma}[{\cite[Theorem 2.1]{Torres12}}] \label{Lem:LinftyContH} If $\alpha>1/2$, then $H^{\alpha}\subset C(\mathbb{R},\mathbb{R}^n)$ and there is a constant $C=C_{\alpha}$ such that $$ \|u\|_{\infty}=\sup_{x\in \mathbb{R}}|u(x)|\leq C\|u\|_{\alpha}. $$ \end{lemma} \begin{remark}\label{Rem:Lp} \rm From Lemma \ref{Lem:LinftyContH}, we know that if $u\in H^{\alpha}$ with $1/2<\alpha<1$, then $u\in L^p(\mathbb{R},\mathbb{R}^n)$ for all $p\in [2,\infty)$, since $$ \int_{\mathbb{R}}|u(x)|^p dx \leq \|u\|^{p-2}_{\infty}\|u\|^2_{L^2}. $$ \end{remark} In what follows, we introduce the fractional space in which we will construct the variational framework of \eqref{FHS}. Let $$ X^{\alpha}=\bigl\{u\in H^{\alpha}: \int_{\mathbb{R}}[|_{-\infty}D^{\alpha}_{t}u(t)|^2+(L(t)u(t),u(t))]dt<\infty\bigr\}, $$ then $X^{\alpha}$ is a reflexive and separable Hilbert space with the inner product $$ \langle u,v \rangle_{X^{\alpha}}=\int_{\mathbb{R}}[(_{-\infty}D^{\alpha}_{t}u(t), {_{-\infty}D}^{\alpha}_{t}v(t))+(L(t)u(t),v(t))]dt $$ and the corresponding norm is $$ \|u\|^2_{X^{\alpha}}=\langle u,u \rangle_{X^{\alpha}}. $$ Similar to Lemma 2.1 in \cite{Torres12}, we have the following conclusion. Its proof is the same as in \cite[Lemma 2.1]{Torres12}, so we omit the details. \begin{lemma}\label{Lem:XcontH} Suppose $L(t)$ satisfies {\rm (A1')}, then $X^{\alpha}$ is continuously embedded in $H^{\alpha}$. \end{lemma} \begin{remark} \rm From Lemmas \ref{Lem:LinftyContH} and \ref{Lem:XcontH}, the embedding of $X^{\alpha}$ into $L^\infty(\mathbb{R},\mathbb{R}^n)$ is continuous. On the other hand, it is obvious that the embedding $X^\alpha\hookrightarrow L^2(\mathbb{R},\mathbb{R}^n)$ is also continuous. Therefore, combining this with Remark \ref{Rem:Lp}, for any $p\in [2,\infty]$, there exists $C_p>0$ such that \begin{equation}\label{eqn:LpContXalpha} \|u\|_{L^p}\leq C_p\|u\|_{X^{\alpha}}. \end{equation} \end{remark} Now we introduce some notation and necessary definitions. Let $\mathcal{B}$ be a real Banach space, $I\in C^1(\mathcal{B},\mathbb{R})$ means that $I$ is a continuously Fr\'echet-differentiable functional defined on $\mathcal{B}$. Recall that $I\in C^1(\mathcal{B},\mathbb{R})$ is said to satisfy the (PS) condition if any sequence $\{u_n\}_{n\in \mathbb{N}}\subset \mathcal{B}$, for which $\{I(u_n)\}_{n\in \mathbb{N}}$ is bounded and $I'(u_n)\to 0$ as $n\to \infty$, possesses a convergent subsequence in $\mathcal{B}$. Moreover, let $B_{r}$ be the open ball in $\mathcal{B}$ with the radius $r$ and centered at $0$ and $\partial B_{r}$ denotes its boundary. Under the conditions of Theorem \ref{Thm:MainTheorem1}, we obtain the existence of the first solution of \eqref{FHS} by using of the following well-known Mountain Pass Theorem, see \cite{Rab86}. \begin{lemma}[{\cite[Theorem 2.2]{Rab86}}] \label{Lem:MountainPass} Let $\mathcal{B}$ be a real Banach space and $I\in C^1(\mathcal{B},\mathbb{R})$ satisfying the {\rm (PS)} condition. Suppose that $I(0)=0$ and \begin{itemize} \item [(A10)] there are constants $\rho$, $\eta>0$ such that $I|_{\partial B_{\rho}}\geq \eta$, and \item [(A11)] there is an $e\in \mathcal{B}\setminus \overline{B}_{\rho}$ such that $I(e)\leq 0$. \end{itemize} Then $I$ possesses a critical value $c\geq \eta$. Moreover $c$ can be characterized as $$ c=\inf_{g\in\Gamma}\max_{s\in[0,1]}I(g(s)), $$ where $$ \Gamma=\left\{g\in C([0,1], \mathcal{B}): g(0)=0,g(1)=e\right\}. $$ \end{lemma} As far as the second one is concerned, we obtain it by minimizing method, which is contained in a small ball centered at $0$, see Step 4 in proof of Theorem \ref{Thm:MainTheorem1} \section{Proof of Theorem \ref{Thm:MainTheorem1}} For this purpose, we establish the corresponding variational framework and then obtain solutions for \eqref{FHS}. Define the functional $I:\mathcal{B}=X^{\alpha}\to \mathbb{R}$ by \begin{equation}\label{eqn:DefinI} \begin{aligned} I(u) &=\int_{\mathbb{R}}\Bigl[\frac{1}{2}|_{-\infty}D_t^{\alpha}u(t)|^2 +\frac{1}{2}(L(t)u(t),u(t))-W(t,u(t))\Bigr]dt\\ &=\dfrac{1}{2}\|u\|^2_{X^{\alpha}}-\int_{\mathbb{R}}W(t,u(t))dt. \end{aligned} \end{equation} Under the conditions of Theorem \ref{Thm:MainTheorem1}, as usual, we see that $I\in C^1(X^{\alpha},\mathbb{R})$, i.e., $I$ is a continuously Fr\'echet-differentiable functional defined on $X^{\alpha}$, see \cite{WuZhang} and \cite{ZhangYuan14} for details. Moreover, we have $$ I'(u)v=\int_{\mathbb{R}}\Bigl[(_{-\infty}D_t^{\alpha}u(t), {_{-\infty}D}_t^{\alpha}v(t))+(L(t)u(t),v(t))-(\nabla W(t,u(t)),v(t))\Bigr]dt $$ for all $u$, $v\in X^{\alpha}$, which yields that \begin{equation}\label{eqn:Iuu} I'(u)u =\|u\|^2_{X^{\alpha}}-\int_{\mathbb{R}}(\nabla W(t,u(t)),u(t))dt. \end{equation} Here, we say that $u\in E^\alpha$ is a solution of \eqref{FHS} if $$ \int_{\mathbb{R}}\Bigl[(_{-\infty}D_t^{\alpha}u(t), {_{-\infty}D}_t^{\alpha}v(t))+(L(t)u(t),v(t))-(\nabla W(t,u(t)),v(t))\Bigr]dt=0 $$ for every $v\in C_0^\infty(\mathbb{R},\mathbb{R}^n)$. To check that the corresponding functional $I(u)$ satisfies the condition (A10) of Lemma \ref{Lem:MountainPass}, the following lemma plays an essential role. \begin{lemma}\label{Lem:psiAB} Let $1<\varrho<2<\theta$, $A, B>0$, and consider the function $$ \Phi_{A,B}(t):=t^2-A t^\varrho-B t^\theta,\quad t\geq 0. $$ Then $\max_{t\geq 0}\Phi_{A,B}(t)>0$ if and only if $$ A^{\theta-2}B^{2-\varrho}0. \end{equation} \end{lemma} The proof of the above lemma is essentially the same as that in \cite[Lemma 3.2]{deFigueiredo}, so we omit it. \begin{lemma}\label{Lem:PS} Under the conditions of Theorem \ref{Thm:MainTheorem1}, $I$ satisfies the {\rm (PS)} condition. \end{lemma} \begin{proof} Assume that $\{u_k\}_{k\in \mathbb{N}} \subset X^\alpha$ is a sequence such that $\left\{I(u_k)\right\}_{k\in \mathbb{N}}$ is bounded and $I'(u_k)\to 0$ as $k\to \infty$. Then there exists a constant $M>0$ such that \begin{equation}\label{eqn:boundeduk} |I(u_k)|\leq M\quad \text{and}\quad\|I'(u_k)\|_{(X^\alpha)^*}\leq M \end{equation} for every $k\in \mathbb{N}$, where $(X^\alpha)^*$ is the dual space of $X^\alpha$. Firstly, we show that $\{u_n\}_{k\in \mathbb{N}}$ is bounded. In fact, in view of \eqref{eqn:W2Estn}, \eqref{eqn:DefinI}, \eqref{eqn:Iuu}, \eqref{eqn:boundeduk}, (A5), (A7), (A8) and \eqref{eqn:LpContXalpha}, we obtain that \begin{align*} M+\frac{M}{\theta} \|u_k\|_{X^\alpha} &\geq I(u_k)-\frac{1}{\theta}I'(u_k)u_k\\ &=\big(\frac{1}{2}-\frac{1}{\theta}\big)\|u_k\|_{X^\alpha}^2 -\int_{\mathbb{R}}\Bigl[W(t,u_k(t))-\frac{1}{\theta}(\nabla W(t,u_k(t)),u_k(t))\Bigr]dt\\ &\geq \big(\frac{1}{2}-\frac{1}{\theta}\big)\|u_k\|_{X^\alpha}^2 -\big(\frac{1}{\varrho}+\frac{1}{\theta}\big)\|c\|_{L^\xi} \|u_k\|_{L^{\varrho \xi^*}}^\varrho\\ &\geq \big(\frac{1}{2}-\frac{1}{\theta}\big)\|u_k\|_{X^\alpha}^2 -C_{\varrho\xi^*}^\varrho \big(\frac{1}{\varrho}+\frac{1}{\theta}\big) \|c\|_{L^\xi}\|u_k\|_{X^\alpha}^\varrho. \end{align*} Since $1<\varrho<2$, the boundedness of $\{u_k\}_{k\in \mathbb{N}}$ follows directly. Then the sequence $\{u_k\}_{k\in \mathbb{N}}$ has a subsequence, again denoted by $\{u_k\}_{k\in \mathbb{N}}$, and there exists $u\in X^\alpha$ such that $$ u_k\rightharpoonup u \quad \text{weakly in } X^\alpha, $$ which yields $$ (I'(u_k)-I'(u))(u_k-u)\to 0\quad \text{as } k\to \infty, $$ and there exists some constant $M_1>0$ such that \begin{equation}\label{eqn:Boundedness} \|u_k\|_{\infty}\leq C_{\infty}\|u_k\|_{X^\alpha} \leq M_1 \quad \text{and} \quad \|u\|_{\infty} \leq C_{\infty}\|u\|_{X^\alpha}\leq M_1 \end{equation} for $k\in \mathbb{N}$. In view of \cite[Lemma 3.1]{WuZhang}, we see that \begin{equation}\label{eqn:NablaW1Con} \int_\mathbb{R} (\nabla W_1(t,u_k(t))-\nabla W_1(t,u(t)),u_k(t)-u(t))dt\to 0 \end{equation} as $k\to \infty$. On the other hand, $c\in L^{\xi}(\mathbb{R},\mathbb{R}^+)$ implies that, for any $\varepsilon>0$, there exists $T>0$ such that \begin{equation}\label{eqn:b-less} \Bigl(\int_{|t|>T}c^{\xi}(t)dt\Bigr)^{1/\xi}<\varepsilon. \end{equation} On account of the continuity of $\nabla W_2(t,u)$ and $u_k\to u$ in $L^{\infty}_{loc}(\mathbb{R},\mathbb{R}^n)$, it follows that there exists $k_0\in \mathbb{N}$ such that \begin{equation}\label{eqn:SmallT} \int_{|t|\leq T}(\nabla W_2(t,u_k(t))-\nabla W_2(t,u(t)), u_k(t)-u(t))dt <\varepsilon \quad \text{for } k\geq k_0. \end{equation} Consequently, joining (A8), \eqref{eqn:LpContXalpha}, \eqref{eqn:Boundedness} and \eqref{eqn:b-less}, we obtain that \begin{equation}\label{eqn:BigT} \begin{aligned} &\int_{|t|>T}(\nabla W_2(t,u_k(t))-\nabla W_2(t,u(t)), u_k(t)-u(t))dt\\ &\leq \int_{|t|>T}|\nabla W_2(t,u_k(t))-\nabla W_2(t,u(t)| |u_k(t)-u(t)|dt\\ &\leq \int_{|t|>T}c(t)(|u_k(t)|^{\varrho-1}+|u(t)|^{\varrho-1}) (|u_k(t)|+|u(t)|)dt\\ &\leq 2 \int_{|t|>T}c(t)(|u_k(t)|^{\varrho}+|u(t)|^{\varrho})dt\\ &\leq 2 \Bigl(\int_{|t|>T}c^{\xi}(t)dt\Bigr)^{1/\xi} (\|u_k\|_{L^{\varrho \xi^*}}^{\varrho}+\|u\|_{L^{\varrho \xi^*}}^{\varrho})\\ &\leq 2 \Bigl(\int_{|t|>T}c^{\xi}(t)dt\Bigr)^{1/\xi} C_{\varrho \xi^*}^{\varrho} (\|u_k\|^{\varrho}_{X^\alpha}+\|u\|^{\varrho}_{X^\alpha})\\ &\leq 4 \varepsilon C_{\varrho \xi^*}^{\varrho} \Bigl(\frac{M_1}{C_{\infty}} \Bigr)^{\varrho}. \end{aligned} \end{equation} Since $\varepsilon>0$ is arbitrary, combining \eqref{eqn:SmallT} with \eqref{eqn:BigT}, we obtain \begin{equation}\label{eqn:NablaW2Con} \int_{\mathbb{R}}(\nabla W_2(t,u_k(t))-\nabla W_2(t,u(t)),u_k(t)-u(t))dt\to 0 \end{equation} as $k\to \infty$. Noting that \begin{align*} &(I'(u_k)-I'(u))(u_k-u)\\ &=\|u_k-u\|_{X^\alpha}^2 -\int_{\mathbb{R}}(\nabla W_1(t,u_k(t))-\nabla W_1(t,u(t)),u_k(t)-u(t))dt\\ &\quad -\int_\mathbb{R}(\nabla W_2(t,u_k(t))-\nabla W_2(t,u(t)),u_k(t)-u(t))dt. \end{align*} Combining this with \eqref{eqn:NablaW1Con} and \eqref{eqn:NablaW2Con}, we deduce that $\|u_k-u\|_{X^\alpha}\to 0$ as $k\to \infty$ and prove that the (PS) condition holds. \end{proof} \begin{proof}[Proof of Theorem \ref{Thm:MainTheorem1}] We divide this proof into four steps. \smallskip \noindent\textbf{Step 1.} It is clear that $I(0)=0$ and that $I\in C^1(X^\alpha,\mathbb{R})$ satisfies the (PS) condition by Lemma \ref{Lem:PS}. \smallskip \noindent\textbf{Step 2.} To show that there exist constants $\rho>0$ and $\eta>0$ such that $I$ satisfies $I|_{\partial B_\rho}\geq \eta>0$; that is, the condition (A10) of Lemma \ref{Lem:MountainPass} holds. To this end, in view of \eqref{eqn:W12Estn} and \eqref{eqn:LpContXalpha}, we have \begin{equation}\label{eqn:WLpestimation} \begin{aligned} \int_0^T W(t,u)dt &\leq \frac{\|a\|_\infty}{\theta}\int_\mathbb{R} |u|^\theta dt+\frac{1}{\varrho}\int_\mathbb{R} c(t)|u|^\varrho dt\\ &\leq\frac{\|a\|_\infty}{\theta}\|u\|_{L^\theta}^\theta+\frac{1}{\varrho}\|c\|_{L^\xi} \|u\|_{L^{\varrho \xi^*}}^\varrho\\ &\leq\frac{\|a\|_\infty C_\theta^\theta}{\theta}\|u\|_{X^\theta}^\theta+\frac{C_{\varrho \xi^*}^\varrho}{\varrho}\|c\|_{L^\xi} \|u\|_{X^\alpha}^\varrho, \end{aligned} \end{equation} which yields \begin{equation}\label{eqn:Iboundedbelow} \begin{aligned} I(u)&=\frac{1}{2}\|u\|_{X^\alpha}^2-\int_0^TW(t,u)dt\\ &\geq \frac{1}{2}\|u\|_{X^\alpha}^2 -\frac{\|a\|_\infty C_\theta^\theta}{\theta}\|u\|_{X^\alpha}^\theta -\frac{C_{\varrho \xi^*}^\varrho}{\varrho}\|c\|_{L^\xi} \|u\|_{X^\alpha}^\varrho \quad \text{for all } u\in X^\alpha. \end{aligned} \end{equation} Applying Lemma \ref{Lem:psiAB} with $$ A=\frac{2\|a\|_\infty C_\theta^\theta}{\theta},\quad B=\frac{2C_{\varrho \xi^*}^\varrho\|c\|_{L^\xi}}{\varrho}, $$ we obtain $$ I(u)\geq \frac{1}{2}\Phi_{A,B}(t_B)>0, $$ provided that $A^{\theta-2}B^{2-\varrho}0$. \smallskip \noindent\textbf{Step 3.} To obtain that there exists an $e\in X^\alpha$ such that $I(e)<0$ with $\|e\|_{X^\alpha}>\rho$, where $\rho$ is defined in Step 2. For this purpose, take $\psi\in X^\alpha$ such that $\psi(t)>0$ on $[0,1]$. In view of \eqref{eqn:DefinI}, \eqref{eqn:W1big1}, (A5) and (A7), for $l\in (0,\infty)$ such that $|l\psi(t)|\geq 1$ for all $t\in [0,1]$, we deduce that \begin{equation}\label{eqn:Ilessthan} \begin{aligned} I(l\psi) &=\frac{l^2}{2}\|\psi\|_{X^\alpha}^2 -\int_\mathbb{R} W(t,l\psi(t))dt\\ &\leq \frac{l^2}{2}\|\psi\|_{X^\alpha}^2 -\int_0^1 W_1(t,l\psi(t))dt\\ &\leq \frac{l^2}{2}\|\psi\|_{X^\alpha}^2-l^\theta \int_0^1 W_1\Bigl(t,\frac{\psi(t)}{|\psi(t)|}\Bigr)|\psi(t)|^\theta dt\\ &\leq \frac{l^2}{2}\|\psi\|_{X^\alpha}^2-m l^\theta \int_0^1 |\psi(t)|^\theta dt, \end{aligned} \end{equation} where $m=\min\{W_1(t,u):t\in [0,1], |u|=1\}$ (on account of (A5), it is obvious that $m>0$). Since $\theta>2$, \eqref{eqn:Ilessthan} implies that $I(l\varphi)=I(e)<0$ for some $l\gg1$ with $\|l\varphi\|_{X^\alpha}>\rho$, where $\rho$ is defined in Step 2. By Lemma \ref{Lem:MountainPass}, $I$ possesses a critical value $c_1\geq\eta>0$ given by $$ c_1=\inf_{g\in\Gamma}\max_{s\in[0,1]}I(g(s)), $$ where $$ \Gamma=\left\{g\in C([0,1], X^\alpha): g(0)=0,\,g(1)=e\right\}. $$ Hence there is $0\neq u_1\in X^\alpha$ such that $$ I(u_1)=c_1\quad \text{and} \quad I'(u_1)=0. $$ That is, the first nontrivial solution of \eqref{FHS} exists. \smallskip \noindent\textbf{Step 4} From \eqref{eqn:Iboundedbelow}, we see that $I$ is bounded from below on $\overline {B_{\rho}(0)}$. Therefore, we can denote $$ c_2=\inf_{\|u\|_{X^\alpha}\leq \rho} I(u), $$ where $\rho$ is defined in Step 1. Then there is a minimizing sequence $\{v_k\}_{k\in \mathbb{N}}\subset \overline{B_{\rho}(0)}$ such that $$ I(v_k)\to c_2 \quad \text{and} \quad I'(v_k)\to 0 $$ as $k\to \infty$. That is, $\{v_k\}_{k\in \mathbb{N}}$ is a (PS) sequence. Furthermore, from Lemma \ref{Lem:PS}, $I$ satisfies the (PS) condition. Therefore, $c_2$ is one critical value of $I$. In what follows, we show that $c_2$ is one nontrivial critical point. Taking $\varphi\in X^\alpha$ such that $\varphi (t)\neq 0$ on $[0,1]$, according to (A5) and (A7), one deduces that, \begin{equation}\label{eqn:Ilessthanzero} \begin{aligned} I(l\varphi) &=\frac{l^2}{2}\|\varphi\|_{X^\alpha}^2 -\int_\mathbb{R} W(t,l\varphi(t))dt\\ &\leq \frac{l^2}{2}\|\varphi\|_{X^\alpha}^2-\int_0^1 W_2(t,l\varphi(t)) dt\\ &\leq \frac{l^2}{2}\|\varphi\|_{X^\alpha}^2-l^\varrho \int_0^1 b(t) |\varphi(t)|^\varrho dt, \quad \forall l\in (0,+\infty). \end{aligned} \end{equation} Since $1<\varrho<2$, \eqref{eqn:Ilessthanzero} implies that $I(l\varphi)<0$ for $l$ small enough such that $\|l\varphi\|_{X^\alpha}\leq \rho$. Therefore, $c_2<0