\documentclass[reqno]{amsart} \usepackage{hyperref} \usepackage{graphicx} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2017 (2017), No. 206, pp. 1--20.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu} \thanks{\copyright 2017 Texas State University.} \vspace{8mm}} \begin{document} \title[\hfilneg EJDE-2017/206\hfil Fractional elliptic systems] {Fractional elliptic systems with nonlinearities of arbitrary growth} \author[E. J. F. Leite \hfil EJDE-2017/206\hfilneg] {Edir Junior Ferreira Leite} \address{Edir Junior Ferreira Leite \newline Departamento de Matem\'{a}tica, Universidade Federal de Vi\c{c}osa, CCE, 36570-000, Vi\c{c}osa, MG, Brazil} \email{edirjrleite@ufv.br} \dedicatory{Communicated by Mokhtar Kirane} \thanks{Submitted June 10, 2017. Published September 7, 2017.} \subjclass[2010]{35J65, 49K20, 35J40} \keywords{Fractional elliptic systems; critical growth; critical hyperbola} \begin{abstract} In this article we discuss the existence, uniqueness and regularity of solutions of the following system of coupled semilinear Poisson equations on a smooth bounded domain $\Omega$ in $\mathbb{R}^n$: \begin{gather*} \mathcal{A}^s u= v^p \quad\text{in }\Omega\\ \mathcal{A}^s v = f(u) \quad\text{in }\Omega\\ u= v=0 \quad\text{on }\partial\Omega \end{gather*} where $s\in (0, 1)$ and $\mathcal{A}^s$ denote spectral fractional Laplace operators. We assume that $1< p<\frac{2s}{n-2s}$, and the function $f$ is superlinear and with no growth restriction (for example $f(r)=re^r$); thus the system has a nontrivial solution. Another important example is given by $f(r)=r^q$. In this case, we prove that such a system admits at least one positive solution for a certain set of the couple $(p,q)$ below the critical hyperbola \[ \frac{1}{p + 1} + \frac{1}{q + 1} = \frac{n - 2s}{n} \] whenever $n > 2s$. For such weak solutions, we prove an $L^\infty$ estimate of Brezis-Kato type and derive the regularity property of the weak solutions. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{remark}[theorem]{Remark} \allowdisplaybreaks \section{Introduction and statement of main result} This work is devoted to the study of existence and uniqueness of solutions for nonlocal elliptic systems on bounded domains which will be described henceforth. The spectral fractional Laplace operator $\mathcal{A}^{s}$ is defined in terms of the Dirichlet spectra of the Laplace operator on $\Omega$. Roughly speaking, if $(\varphi_k)$ denotes a $L^2$-orthonormal basis of eigenfunctions corresponding to eigenvalues $(\lambda_k)$ of the Laplace operator with zero Dirichlet boundary values on $\partial \Omega$, then the operator $\mathcal{A}^s$ is defined as $\mathcal{A}^{s} u = \sum_{k=1}^\infty c_k \lambda_k^s \varphi_k$, where $c_k$, $k \geq 1$, are the coefficients of the expansion $u = \sum_{k=1}^\infty c_k \varphi_k$. A closely related to (but different from) the spectral fractional Laplace operator $\mathcal{A}^{s}$ is the restricted fractional Laplace operator $(-\Delta)^s$, see \cite{compare, rafaella}. This is defined as \[ (-\Delta)^{s}u(x) = C(n,s)\operatorname{P.V.} \int_{\mathbb{R}^{n}}\frac{u(x)-u(y)}{| x-y|^{n+2s}}\,dy\, , \] for all $x \in \mathbb{R}^{n}$, where P.V. denotes the principal value of the first integral and \[ C(n,s) = \Big(\int_{\mathbb{R}^{n}}\frac{1-\cos(\zeta_{1})}{|\zeta|^{n+2s}}\,d\zeta\Big)^{-1} \] with $\zeta = (\zeta_1, \ldots, \zeta_n) \in \mathbb{R}^n$. We remark that $(-\Delta)^{s}$ is a nonlocal operator on functions compactly supported in $\mathbb{R}^n$, i.e., to check whether the equation holds at a point, information about the values of the function far from that point is needed. Fractional Laplace operators arise naturally in several different areas such as Probability, Finance, Physics, Chemistry and Ecology, see \cite{A, bucur}. These operators have attracted special attention during the last decade. An extension for spectral fractional operator was devised by Cabr\'{e} and Tan \cite{CT} and Capella, D\'{a}vila, Dupaigne, and Sire \cite{CDDS} (see Br\"{a}ndle, Colorado, de Pablo, and S\'{a}nchez \cite{BrCPS} and Tan \cite{T} also). Thanks to these advances, the boundary fractional problem \begin{equation} \begin{gathered} \mathcal{A}^{s} u = u^p \quad\text{in }\Omega\\ u = 0 \quad\text{on }\partial\Omega \end{gathered} \end{equation} has been widely studied on a smooth bounded open subset $\Omega$ of $\mathbb{R}^n$, $n\geq 2$, $s \in (0,1)$ and $p > 0$. Particularly, a priori bounds and existence of positive solutions for subcritical exponents ($p < \frac{n + 2s}{n - 2s}$) has been proved in \cite{BrCPS, CT, choi, CK, T} and nonexistence results has also been proved in \cite{BrCPS, tan1, T} for critical and supercritical exponents ($p \geq \frac{n + 2s}{n - 2s}$). The regularity result has been proved in \cite{CDDS, T, yang}. When $s = 1/2$, Cabr\'{e} and Tan \cite{CT} established the existence of positive solutions for equations having nonlinearities with the subcritical growth, their regularity, the symmetric property, and a priori estimates of the Gidas-Spruck type by employing a blow-up argument along with a Liouville type result for the square root of the Laplace operator in the half-space. Then \cite{T} has the analogue to $1/2\begin{cases} 2 &\text{if } p> 1\\ 1+\frac{1}{p} &\text{if } p\leq1 \end{cases} \end{equation} and $r_0\geq 0$ such that $\theta F(r)\leq f(r)r$ for all $| r|\geq r_0$ and \[ f(r)=\begin{cases} o(r) &\text{if } p> 1\\ o(r^{1/p}) &\text{if } p\leq1 \end{cases} \] for $r$ near $0$. Then the system \begin{equation}\label{1} \begin{gathered} \mathcal{A}^s u= v^p \quad\text{in }\Omega\\ \mathcal{A}^s v = f(u) \quad\text{in }\Omega\\ u= v=0 \quad\text{on }\partial\Omega \end{gathered} \end{equation} has a nontrivial weak solution. \end{theorem} Note that if $2\leq n<4s$ then $n=2$ and $s\in\left(\frac{1}{2},1\right)$ or $n=3$ and $s\in\left(\frac{3}{4},1\right)$. \begin{theorem} \label{teo2} Suppose that $n\geq 4s$, $04s\\ 04s\\ 0

1$, a priori bounds and existence of positive solutions of \eqref{2} have been derived in \cite{choi, E2} provided that $p, q$ satisfy \begin{equation} \label{Hip} \frac{1}{p+1} + \frac{1}{q+1} > \frac{n - 2s}{n}\,. \end{equation} \begin{remark} \label{rmk1.2} \rm In the case when $n\leq 4s$, the above Theorems cover the remaining cases below the critical hyperbola and when $pq\neq 1$. In the case when $n>4s$, Figure \ref{fig1} exemplifies the region that the above theorem covers. \end{remark} \begin{figure}[ht] \begin{center} \includegraphics[width=0.7\textwidth]{fig1} \end{center} \caption{Existence range of couples $(p,q)$ when $n>4s$.} \label{fig1} \end{figure} \begin{remark} \label{rmk1.3} \rm For such weak solutions, we prove an $L^\infty$ estimate of Brezis-Kato type and derive the regularity property of the weak solutions based on the results obtained in \cite{T}. \end{remark} For $s = 1$, the problem \eqref{2} and a number of its generalizations have been widely investigated in the literature, see for instance the survey \cite{DG} and references therein. Specifically, notions of sublinearity, superlinearity and criticality (subcriticality, supercriticality) have been introduced in \cite{FM, Mi1, Mi2, SZ1}. In fact, the behavior of \eqref{2} is sublinear when $pq < 1$, superlinear when $pq > 1$ and critical (subcritical, supercritical) when $n \geq 3$ and $(p,q)$ is on (below, above) the hyperbola, known as critical hyperbola, \[ \frac{1}{p+1}+\frac{1}{q+1}=\frac{n-2}{n}\, . \] When $pq = 1$, its behavior is resonant and the corresponding eigenvalue problem has been addressed in \cite{marcos}. The sublinear case has been studied in \cite{FM} where the existence and uniqueness of positive classical solution is proved. The superlinear-subcritical case has been covered in the works \cite{CFM}, \cite{DF}, \cite{DR} and \cite{vander} where the existence of at least one positive classical solution is derived. Lastly, the nonexistence of positive classical solutions has been established in \cite{Mi1} on star-shaped domains. When $0 < s < 1$ and $p, q > 0$, existence of positive solutions of \eqref{2} for the restricted fractional Laplace operator $(-\Delta)^s$ have been derived in \cite{E2, EM1} provided that $pq\neq 1$ and $(p, q)$ satisfies \eqref{Hip}. Related systems have been studied with topological methods. We refer to the work \cite{EM} for systems involving different operators $(-\Delta)^{s}$ and $(-\Delta)^{t}$ in each one of equations. The rest of paper is organized into five sections. In Section 2 we briefly recall some definitions and facts related to spectral fractional Laplace operator. In Section 3, we prove the case $p> 1$ of Theorem \ref{teo1} by applying the Strongly Indefinite Functional Theorem of Li-Willem. Then, we prove the case $p\leq 1$ by using the mountain pass theorem of Ambrosetti-Rabinowitz. In Section 4, we prove the case $pq<1$ of Theorem \ref{teo2} by using a direct minimization approach, Hopf lemma and maximum principles. Next we establish the remaining cases by using the mountain pass theorem. In Section 5, we establish regularity property of the weak solutions of system \eqref{1} based on the results obtained in \cite{T}. Finally we establish the Brezis-Kato type result and derive the regularity of solutions to \eqref{2}. \section{Preliminaries} In this section we briefly recall some definitions and facts related to spectral fractional Laplace operator. The spectral fractional Laplace operator $\mathcal{A}^s$ is defined as follows. Let $\varphi_k$ be an eigenfunction of $-\Delta$ given by \begin{equation} \begin{gathered} -\Delta\varphi_k = \lambda_k\varphi_k \quad\text{in }\Omega \\ \varphi_k = 0 \quad\text{on }\partial\Omega, \end{gathered} \end{equation} where $\lambda_k$ is the corresponding eigenvalue of $\varphi_k,0<\lambda_1<\lambda_2\leq\lambda_3\leq\cdots\leq\lambda_k\to +\infty$. Then, $\{\varphi_k\}_{k=1}^{\infty}$ is an orthonormal basis of $L^2(\Omega)$ satisfying \[ \int_{\Omega}\varphi_j\varphi_kdx=\delta_{j,k}. \] We define the operator $\mathcal{A}^s$ for any $u\in C^\infty_0(\Omega)$ by \begin{equation}\label{lapla} \mathcal{A}^su=\sum_{k=1}^{\infty}\lambda_k^s\xi_k\varphi_k, \end{equation} where \[ u=\sum_{k=1}^{\infty}\xi_k\varphi_k\quad\text{and}\quad \xi_k=\int_{\Omega}u\varphi_k\,dx. \] This operator is defined on a Hilbert space \[ \Theta^{s}(\Omega)=\{u=\sum_{k=1}^{\infty}u_k\varphi_{k} \in L^2(\Omega)\mid\sum_{k=1}^{\infty}\lambda^s_{k}| u_k|^2<+\infty\} \] with values in its dual $\Theta^{s}(\Omega)'$. Thus the inner product of $\Theta^{s}(\Omega)$ is given by \[ \langle u,v\rangle_{\Theta^{s}(\Omega)} =\int_{\Omega}\mathcal{A}^{s/2}u\mathcal{A}^{s/2}v\,dx =\int_{\Omega}u\mathcal{A}^sv\,dx=\int_{\Omega}v\mathcal{A}^sudx. \] We denote by $\|\cdot\|_{\Theta^{s}}$ the norm derived from this inner product. We remark that $\Theta^{s}(\Omega)'$ can be described as the completion of the finite sums of the form \[ f=\sum_{k=1}^{\infty}c_k\varphi_{k} \] with respect to the dual norm \[ \| f\|_{\Theta^{s}(\Omega)'}=\sum_{k=1}^{\infty}(\lambda_{k}^s)^{-1}| c_k|^2 =\|\mathcal{A}^{-s/2}f\|_{L^2}^2=\int_{\Omega}f\mathcal{A}^{-s}f\,dx \] and it is a space of distributions. Moreover, the operator $\mathcal{A}^s$ is an isomorphism between $\Theta^{s}(\Omega)$ and $\Theta^{s}(\Omega)'\simeq\Theta^{s}(\Omega)$, given by its action on the eigenfunctions. If $u,v\in\Theta^{s}(\Omega)$ and $f = \mathcal{A}^su$ we have, after this isomorphism, \[ \langle f, v\rangle_{\Theta^{s}(\Omega)'\times\Theta^{s}(\Omega)} = \langle u, v\rangle_{\Theta^{s}(\Omega)\times\Theta^{s}(\Omega)} =\sum_{k=1}^{\infty}\lambda^s_{k}u_kv_k. \] If it also happens that $f\in L^{2}(\Omega)$, then clearly we obtain \[ \langle f, v\rangle_{\Theta^{s}(\Omega)'\times\Theta^{s}(\Omega)} =\int_{\Omega}fv\,dx. \] We have $\mathcal{A}^{-s}:\Theta^{s}(\Omega)'\to\Theta^{s}(\Omega)$ can be written as \[ \mathcal{A}^{-s}f(x)=\int_{\Omega}G_{\Omega}(x,y)f(y)dy, \] where $G_{\Omega}$ is the Green function of operator $\mathcal{A}^s$ (see \cite{Green, ros176}). It is known that \[ \Theta^s(\Omega) =\begin{cases} L^2(\Omega) &\text{if } s=0\\ H^{s}(\Omega)=H^{s}_0(\Omega) &\text{if }s\in(0,1/2)\\ H^{1/2}_{00}(\Omega) &\text{if }s=1/2\\ H^{s}_0(\Omega) &\text{if }s\in(1/2,1]\\ H^{s}(\Omega)\cap H^{1}_0(\Omega) &\text{if }s\in(1,2], \end{cases} \] where $H^{1/2}_{00}(\Omega):=\{u\in H^{1/2}(\Omega): \int_{\Omega}\frac{u^2(x)}{d(x)}dx<+\infty\}$, (see \cite{vander}). Observe that the injection $\Theta^s(\Omega)\hookrightarrow H^{s}(\Omega)$ is continuous. By the Sobolev imbedding theorem (see \cite{DD}) we therefore have continuous imbeddings $\Theta^s(\Omega)\subset L^{p+1}(\Omega)$ if $p+1\leq\frac{2n}{n-2s}$ and these imbedding are compact if $p+1<\frac{2n}{n-2s}$ for $0\frac{1}{2}, \] where $C(\overline{\Omega})$ is a Banach space with the norm \[ \| u\|_C=\sup_{\Omega}| u|. \] For $01$. \subsection{The case $p>1$} We define the product spaces \[ E^{\alpha}(\Omega)=\Theta^{\alpha}(\Omega)\times\Theta^{2s-\alpha}(\Omega). \] For $0<\alpha<2s$ the space $E^{\alpha}(\Omega)$ is a Hilbert space with inner product \[ \langle(u_1,v_1),(u_2,v_2)\rangle_{E^{\alpha}(\Omega)} =\langle\mathcal{A}^{\alpha/2}u_1,\mathcal{A}^{\alpha/2}u_2\rangle_{L^{2}(\Omega)} +\langle\mathcal{A}^{s-\alpha/2}v_1,\mathcal{A}^{s-\alpha/2}v_2 \rangle_{L^{2}(\Omega)}. \] We denote by $\|\cdot\|_{E}$ the norm derived from this inner product, i.e, \[ \|(u,v)\|_{E}=\left(\| u\|^2_{\Theta^{\alpha}} +\| v\|^2_{\Theta^{2s-\alpha}}\right)^{1/2}. \] We also have $\mathcal{A}^s:\Theta^\alpha(\Omega)\to\Theta^{\alpha-2s}(\Omega)$ is an isomorphism, see \cite{vander}. Hence $$ \begin{pmatrix} 0& \mathcal{A}^{s} \\ \mathcal{A}^{s} & 0 \end{pmatrix}: E^\alpha(\Omega)\to\Theta^{-\alpha}\times\Theta^{\alpha-2s}(\Omega) =E^\alpha(\Omega)' $$ is an isometry. We consider the Lagrangian \begin{equation}\label{func} \mathcal{J}(u,v) =\int_{\Omega}\mathcal{A}^{s/2}u\mathcal{A}^{s/2}v\,dx -\int_{\Omega}\Big(\frac{1}{p+1}| v|^{p+1}+F(u)\Big)dx, \end{equation} i.e., a strongly indefinite functional. The Hamiltonian is given by \begin{equation}\label{2.2} \mathcal{H}(u,v)=\int_{\Omega}\Big(\frac{1}{p+1}| v|^{p+1}+F(u)\Big)dx. \end{equation} The quadratic part can again be written as \[ A(u,v)=\frac{1}{2}\langle L(u,v),(u,v)\rangle_{E^{\alpha}(\Omega)} =\int_{\Omega}\mathcal{A}^{\alpha/2}u\mathcal{A}^{s-\alpha/2}v\,dx =\int_{\Omega}\mathcal{A}^{s/2}u\mathcal{A}^{s/2}v\,dx, \] where $$ L=\begin{pmatrix} 0& \mathcal{A}^{s-\alpha} \\ \mathcal{A}^{\alpha-s} & 0 \end{pmatrix} $$ is bounded and self-adjoint. Introducing the ``diagonals" \[ E^+=\{(u,\mathcal{A}^{\alpha-s}u):u\in\Theta^\alpha(\Omega)\}\text{ and } E^-=\{(u,-\mathcal{A}^{\alpha-s}u):u\in\Theta^\alpha(\Omega)\} \] we have \[ E^{\alpha}(\Omega)=E^+\oplus E^-. \] An orthonormal basis of $E^\alpha(\Omega)$ is given by \[ \big\{\frac{1}{\sqrt{2}}(\lambda_k^{-\alpha/2}\varphi_k, \pm\lambda_k^{\alpha/2-s}\varphi_k):k=1,2,\dots\big\}. \] The derivative of $A(u,v)$ defines a bilinear form \begin{equation}\label{1.27} B((u_1,v_1),(u_2,v_2))=A'(u_1,v_1)(u_2,v_2) =\langle L(u_1,v_1),(u_2,v_2)\rangle_{E^{\alpha}(\Omega)}, \end{equation} where $(u_1,v_1),(u_2,v_2)\in E^{\alpha}(\Omega)$ with \begin{equation}\label{1.28} A(u_1,v_1)=\frac{1}{2}B((u_1,v_1),(u_1,v_1))\quad\text{and}\quad B((u_1,v_1)^+,(u_1,v_1)^-)=0. \end{equation} We will give the choice of $\alpha$ in the following lemma. \begin{lemma}\label{lema3.1} Let $1 < p<2s/(n-2s)$. Then there exists a parameter $0<\alpha<2s$ such that the following embeddings are continuous and compact, \[ \Theta^{2s-\alpha}(\Omega)\subset L^{p+1}(\Omega) \quad\text{and}\quad \Theta^\alpha(\Omega)\subset C(\overline{\Omega}). \] \end{lemma} \begin{proof} Note that $\Theta^{2s-\alpha}(\Omega)\subset L^{q}(\Omega)$ compactly, if $q<\frac{2n}{n-4s+2\alpha}$ and $\Theta^\alpha(\Omega)\subset C(\overline{\Omega})$ compactly, if \[ \frac{\alpha}{n}>\frac{1}{2}, \] see \cite{DD}. We have $p+1<\frac{n}{n-2s}$. Thus if $\alpha>\frac{n}{2}$, then $p+1<\frac{2n}{n-4s+2\alpha}$. For $n=2$, we have $s\in\left(\frac{1}{2},1\right)$. In this case the result is valid for all $1<\alpha<2s$. For $n=3$, we have $s\in(3/4,1)$. In this case the result is valid for all $\frac{3}{2}<\alpha<2s$. This ends the proof. \end{proof} \begin{remark} \label{rmk3.1} \rm Note that $\alpha-s>0$. Thus $\Theta^\alpha(\Omega)\hookrightarrow\Theta^s(\Omega)$ is compact. \end{remark} The functional $\mathcal{J}(u, v): E^\alpha(\Omega)\to\mathbb{R}$ is strongly indefinite near zero, in the sense that there exist infinite dimensional subspaces $E^+$ and $E^-$ with $E^+\oplus E^- = E^\alpha(\Omega)$ such that the functional is (near zero) positive definite on $E^+$ and negative definite on $E^-$. Li-Willem \cite{LI} prove the following general existence theorem for such situations, which can be applied in our case. \begin{theorem}[Li-Willem, 1995] \label{teo2.4} Let $\Phi:E\to\mathbb{R}$ be a strongly indefinite $C^1$-functional satisfying \begin{itemize} \item[(i)] $\Phi$ has a local linking at the origin, i.e. for some $r > 0$: \[ \Phi(z)\geq 0 \text{ for }z\in E^+,\text{ } \| z\|_E\leq r \text{ and }\Phi(z)\leq 0,\text{ for }z\in E^-,\text{ } \| z\|_E\leq r; \] \item[(ii)] $\Phi$ maps bounded sets into bounded sets; \item[(iii)] let $E^+_n$ be any $n$-dimensional subspace of $E^+$; then $\Phi(z)\to -\infty$ as $\| z\|\to +\infty$, $z\in E^+_n\oplus E^-$; \item[(iv)] $\Phi$ satisfies the Palais-Smale condition $(PS)$ (Li-Willem \cite{LI} require a weaker $(PS^\ast)$-condition, however, in our case the classical $(PS)$ condition will be satisfied). \end{itemize} Then $\Phi$ has a nontrivial critical point. \end{theorem} We now verify that the functional defined in \eqref{func} satisfies the assumptions of this theorem. First, it is clear, with the choice of $\alpha$ (Lemma \ref{lema3.1}), that $\mathcal{J}(u,v)$ is a $C^1$-functional on $E^\alpha(\Omega)$. We show that the condition (i) of Theorem \ref{teo2.4} is satisfied. It is easy to see that $\mathcal{J}(u,v)$ has a local linking with respect to $E^+$ and $E^-$ at the origin. Now the condition (ii) of Theorem \ref{teo2.4}. Let $B\subset E^\alpha(\Omega)$ be a bounded set, i.e. $\| u\|_{\Theta^\alpha}\leq c$, $\| v\|_{\Theta^{2s-\alpha}}\leq c$, for all $(u,v)\in B$. Then \begin{align*} |\mathcal{J}(u,v)| &\leq \|\mathcal{A}^{\alpha/2}u\|_{L^2}\|\mathcal{A}^{s-\alpha/2}v\|_{L^2} +\int_{\Omega}| v|^{p+1}dx + \int_{\Omega}| f(u)|\,dx\\ &\leq \| u\|_{\Theta^\alpha}\| v\|_{\Theta^{2s-\alpha}} +c\| v\|^{p+1}_{\Theta^{2s-\alpha}} + |\Omega|\sup\{| f(u(x))|:x\in\Omega\}\leq C. \end{align*} We show that the condition (iii) of Theorem \ref{teo2.4} is satisfied. Let $z_k = z^+_k + z^-_k\in E^+_n\oplus E^-$ denote a sequence with $\| z_k\|_E\to +\infty$. Thus, $z_k$ may be written as \[ z_k = (u_k,\mathcal{A}^{\alpha-s}u_k) + (w_k,-\mathcal{A}^{\alpha-s}w_k), \] with $u_k\in \Theta^\alpha_n(\Omega)$, $w_k\in \Theta^\alpha(\Omega)$, where $\Theta^\alpha_n(\Omega)$ denotes an $n$-dimensional subspace of $\Theta^\alpha(\Omega)$. Thus, the functional $\mathcal{J}(z_k)$ takes the form \begin{align*} \mathcal{J}(z_k) &= \int_\Omega \mathcal{A}^{\alpha/2}u_k\mathcal{A}^{s-\alpha/2} \mathcal{A}^{\alpha-s}u_kdx -\int_\Omega \mathcal{A}^{\alpha/2} w_k\mathcal{A}^{s-\alpha/2}\mathcal{A}^{\alpha-s}w_kdx \\ & \quad -\frac{1}{p+1}\int_\Omega|\mathcal{A}^{\alpha -s}(u_k-w_k)|^{p+1}dx -\int_\Omega F(u_k-w_k)dx\\ &= \int_\Omega |\mathcal{A}^{\alpha/2}u_k|^2dx -\int_\Omega |\mathcal{A}^{\alpha/2}w_k|^2dx -\frac{1}{p+1}\int_\Omega|\mathcal{A}^{\alpha -s}(u_k-w_k)|^{p+1}dx\\ &\quad -\int_\Omega F(u_k-w_k)dx. \end{align*} Note that $\| z_k\|_E\to\infty$ if and only if \[ \int_\Omega|\mathcal{A}^{\alpha/2}u_k|^2dx + \int_\Omega|\mathcal{A}^{\alpha/2}w_k|^2dx =\| u_k\|^2_{\Theta^\alpha}+\| w_k\|^2_{\Theta^\alpha}\to\infty. \] Now, if \begin{itemize} \item[(1)] $\| u_k\|^2_{\Theta^\alpha}\leq c$, then $\| w_k\|^2_{\Theta^\alpha}\to\infty$, and then $\mathcal{J}(z_k)\to -\infty$; \item[(2)] $\| u_k\|^2_{\Theta^\alpha}\to +\infty$, then using the fact that $\alpha-s>0$ and $p > 1$ there exists $c, c_1$ and $c_2$ positive constants such that \[ \int_\Omega|\mathcal{A}^{\alpha-s}(u_k - w_k)|^{p+1}dx \geq c\Big(\int_\Omega|\mathcal{A}^{\alpha-s}(u_k - w_k)|^2dx\Big)^{\frac{p+1}{2}} \geq c_1\| u_k-w_k\|_{L^2}^{p+1} \] and \[ \int_\Omega F(u_k + w_k)dx\geq c_2\int_\Omega| u_k + w_k|^{p+1}dx-d \geq c_1\| u_k+w_k\|_{L^2}^{p+1}-d \] and hence we obtain the estimate \[ \mathcal{J}(z_k)\leq\frac{1}{2}\| u_k\|^2_{\Theta^\alpha} -c_1\left(\| u_k-w_k\|_{L^2}^{p+1}+\| u_k+w_k\|_{L^2}^{p+1}\right)+d. \] \end{itemize} Since $\phi(t) = t^{p+1}$ is convex, we have $\frac{1}{2}(\phi(t)+\phi(r))\geq\phi\left(\frac{1}{2}(r+t)\right)$, and hence \begin{align*} \mathcal{J}(z_k) & \leq \frac{1}{2}\| u_k\|^2_{\Theta^\alpha}-c_1 \frac{1}{2^p}\left(\| u_k-w_k\|_{L^2}+\| u_k+w_k\|_{L^2}\right)^{p+1}+d \\ & \leq \frac{1}{2}\| u_k\|^2_{\Theta^\alpha}-c_1\frac{1}{2^p}\| u_k\|^{p+1}_{L^2}+d. \end{align*} Since on $\Theta_n^\alpha(\Omega)$ the norms $\| u_k\|_{\Theta^\alpha}$ and $\| u_k\|_{L^2}$ are equivalent (see \cite{vander}), we conclude that also in this case $\mathcal{J}(z_k)\to -\infty$. Finally, the condition (iv) of Theorem \ref{teo2.4}. Let $(z_n)\subset E^\alpha(\Omega)$ denote a $(PS)$-sequence, i.e. such that \begin{equation}\label{2.5} |\mathcal{J}(z_n)|\to c,\text{ and }|(\mathcal{J}'(z_n),\eta)| \leq\epsilon_n\|\eta\|_E,\quad \forall\eta\in E^\alpha(\Omega),\text{ and }\epsilon_n\to 0. \end{equation} \begin{lemma} The $(PS)$-sequence $(z_n)$ is bounded in $E^\alpha(\Omega)$. \end{lemma} \begin{proof} By \eqref{2.5} we have for $z_n=(u_n, v_n)$, \[ \mathcal{J}(u_n, v_n) = \int_\Omega\mathcal{A}^{\alpha/2}u_n \mathcal{A}^{s-\alpha/2}v_n\,dx -\frac{1}{p+1}\int_\Omega v_n^{p+1}\,dx - \int_\Omega F(u_n)dx\to c \] and \begin{equation}\label{2.7} |\mathcal{J}'(u_n, v_n)(\varphi,\phi)|\leq\epsilon_n\|(\varphi,\phi)\|_E \leq\epsilon_n(\| \varphi\|_{\Theta^\alpha}+\| \phi\|_{\Theta^{2s-\alpha}}), \end{equation} where \begin{align*} \mathcal{J}'(u_n, v_n)(\varphi,\phi) &=\int_\Omega\mathcal{A}^{\alpha/2}u_n\mathcal{A}^{s-\alpha/2}\phi\,dx +\int_\Omega\mathcal{A}^{s-\alpha/2}v_n\mathcal{A}^{\alpha/2}\varphi\,dx \\ &\quad -\int_\Omega v^p_n\phi\,dx -\int_\Omega f(u_n)\varphi\,dx. \end{align*} Choosing $(\varphi,\psi) = (u_n, v_n)\in \Theta^{\alpha}(\Omega) \times\Theta^{2s-\alpha}(\Omega)$ we obtain by \eqref{2.7}, \[ 2\int_\Omega\mathcal{A}^{\alpha/2}u_n\mathcal{A}^{s-\alpha/2}v_ndx -\int_\Omega v_n^{p+1}\,dx - \int_\Omega f(u_n)u_n\,dx \leq\epsilon_n(\| u_n\|_{\Theta^\alpha}+\| v_n\|_{\Theta^{2s-\alpha}}) \] and subtracting this from $2\mathcal{J}(u_n, v_n)$ we obtain, using assumption \eqref{assumption} of Theorem \ref{teo1} \[ \big(1-\frac{2}{p+1}\big)\int_\Omega v_n^{p+1}\,dx +\big(1-\frac{2}{\theta}\big) \int_\Omega f(u_n)u_n\,dx \leq C+\epsilon_n(\| u_n\|_{\Theta^\alpha}+\| v_n\|_{\Theta^{2s-\alpha}}) \] and thus \begin{gather}\label{2.10} \int_\Omega v_n^{p+1}\,dx \leq C +\epsilon_n(\| u_n\|_{\Theta^\alpha} +\| v_n\|_{\Theta^{2s-\alpha}}), \\ \label{2.11} \int_\Omega f(u_n)u_n\,dx\leq C +\epsilon_n(\| u_n\|_{\Theta^\alpha} +\| v_n\|_{\Theta^{2s-\alpha}}). \end{gather} Choosing $(\varphi,\phi) = (0,\mathcal{A}^{\alpha-s}u_n)\in \Theta^{\alpha} (\Omega)\times\Theta^{2s-\alpha}(\Omega)$ in \eqref{2.7} we obtain \[ \int_\Omega|\mathcal{A}^{\alpha/2}u_n|^2\,dx \leq\int_\Omega v^p_n\mathcal{A}^{\alpha-s}u_ndx +\epsilon_n\|\mathcal{A}^{\alpha-s}u_n\|_{\Theta^{2s-\alpha}} \] and hence by H\"{o}lder inequality, \[ \| u_n\|^2_{\Theta^{\alpha}}=\|\mathcal{A}^{\alpha/2}u_n\|^2_{L^2} \leq \Big(\int_\Omega| v_n|^{p+1}\,dx\Big)^{\frac{p}{p+1}} \Big(\int_\Omega |\mathcal{A}^{\alpha-s}u_n|^{p+1}dx\Big)^{\frac{1}{p+1}} +\epsilon_n\| u_n\|_{\Theta^{\alpha}}. \] Noting that \[ \Big(\int_\Omega|\mathcal{A}^{\alpha-s}u_n|^{p+1}\,dx\Big)^{\frac{1}{p+1}} \leq c\| \mathcal{A}^{\alpha-s}u_n\|_{\Theta^{2s-\alpha}} = c\| \mathcal{A}^{\alpha/2}u_n\|_{L^2} = c\| u_n\|_{\Theta^{\alpha}} \] and using \eqref{2.10}, we obtain \[ \| u_n\|^2_{\Theta^{\alpha}} \leq [C + \epsilon_n(\| u_n\|_{\Theta^\alpha} +\| v_n\|_{\Theta^{2s-\alpha}})]^{p/(p+1)}\cdot c \| u_n\|_{\Theta^\alpha} +\epsilon_n\| u_n\|_{\Theta^{\alpha}}\,; \] therefore \begin{equation}\label{2.12} \| u_n\|_{\Theta^{\alpha}}\leq C + \epsilon_n(\| u_n\|_{\Theta^\alpha} +\| v_n\|_{\Theta^{2s-\alpha}})^{p/(p+1)}. \end{equation} As above we note that $\mathcal{A}^{s-\alpha}v_n\in\Theta^\alpha(\Omega)$, and thus, choosing $(\varphi,\psi) =(\mathcal{A}^{s-\alpha}v_n, 0)\in \Theta^\alpha(\Omega) \Theta^{2s-\alpha}(\Omega)$ in \eqref{2.7} we obtain \begin{align*} \int_\Omega| \mathcal{A}^{s-\alpha/2}v_n|^2dx &\leq \int_\Omega f(u_n)\mathcal{A}^{s-\alpha}v_n\,dx + \epsilon_n\|\mathcal{A}^{s-\alpha}v_n\|_{\Theta^\alpha}\\ &\leq \|\mathcal{A}^{s-\alpha}v_n\|_\infty\int_\Omega| f(u_n)|\,dx + \epsilon_n\| v_n\|_{\Theta^{\alpha}}. \end{align*} Using that $\|\mathcal{A}^{s-\alpha}v_n\|_{\Theta^\alpha} = \|\mathcal{A}^{s-\alpha/2}v_n\|_{L^2} = \| v_n\|_{\Theta^{2s-\alpha}}$, and the fact that $\Theta^{\alpha}(\Omega)\subset C(\overline{\Omega})$ we then obtain, using \eqref{2.11}, \begin{equation} \label{2.13} \begin{aligned} \| v_n\|_{\Theta^\alpha} &\leq c\int_\Omega | f(u_n)|\,dx + \epsilon_n \\ &= \int_{[| u_n|\leq s_0]}\max_{| t|\leq s_0}| f(t)|\,dx + \int_{[| u_n|> s_0]} f(u_n)u_n\,dx + \epsilon_n\\ &\leq C + \epsilon_n(\| u_n\|_{\Theta^{\alpha}} + \| v_n\|_{\Theta^{2s-\alpha}}). \end{aligned} \end{equation} Joining \eqref{2.12} and \eqref{2.13} we finally get \[ \| u_n\|_{\Theta^{\alpha}} + \| v_n\|_{\Theta^{2s-\alpha}} \leq C + 2\epsilon_n(\| u_n\|_{\Theta^{\alpha}} + \| v_n\|_{\Theta^{2s-\alpha}}). \] Thus, $\| u_n\|_{\Theta^{\alpha}} + \| v_n\|_{\Theta^{2s-\alpha}}$ is bounded. \end{proof} With this it is now possible to complete the proof of the $(PS)$-condition: since $\| u_n\|_{\Theta^\alpha}$ is bounded, we find a weakly convergent subsequence $u_n\rightharpoonup u$ in $\Theta^{\alpha}(\Omega)$. Since the mappings $\mathcal{A}^{\alpha/2}: \Theta^{\alpha}(\Omega)\to L^2(\Omega)$ and $\mathcal{A}^{\alpha/2-s}: L^2(\Omega)\to \Theta^{2s-\alpha}(\Omega)$ are continuous isomorphisms, we obtain $\mathcal{A}^{\alpha/2}(u_n - u)\rightharpoonup 0$ in $L^2(\Omega)$ and $\mathcal{A}^{\alpha-s}(u_n - u) \rightharpoonup 0$ in $\Theta^{2s-\alpha}(\Omega)$. Since $\Theta^{2s-\alpha}(\Omega)\subset L^{p+1}(\Omega)$ compactly, we conclude that $\mathcal{A}^{\alpha-s}(u_n - u)\to 0$ strongly in $L^{p+1}(\Omega)$. Similarly, we find a subsequence of $(v_n)$ which is weakly convergent in $\Theta^{2s-\alpha}(\Omega)$ and such that $v^p_n$ is strongly convergent in $L^{\frac{p+1}{p}}(\Omega)$. Choosing $(\varphi,\phi) = (0,\mathcal{A}^{\alpha-s}(u_n - u))\in \Theta^{\alpha} (\Omega)\times\Theta^{2s-\alpha}(\Omega)$ in \eqref{2.7} we thus conclude \[ \int_\Omega\mathcal{A}^{\alpha/2}u_n\mathcal{A}^{\alpha/2}(u_n - u)\,dx \leq\int_\Omega v^p_n\mathcal{A}^{\alpha-s}(u_n - u)\,dx + \epsilon_n\|\mathcal{A}^{\alpha-s}(u_n - u)\|_{\Theta^{2s-\alpha}}. \] By the above considerations, the right-hand-side converges to $0$, and thus \[ \int_\Omega|\mathcal{A}^{\alpha/2}u_n|^2\,dx \to\int_\Omega|\mathcal{A}^{\alpha/2}u|^2\,dx. \] Thus, $u_n\to u$ strongly in $\Theta^{\alpha}(\Omega)$. To obtain the strong convergence of $(v_n)$ in $\Theta^{2s-\alpha}(\Omega)$, one proceeds similarly: as above, one finds a subsequence $(v_n)$ converging weakly in $\Theta^{2s-\alpha}(\Omega)$ to $v$, and then $\mathcal{A}^{s-\alpha}v_n\rightharpoonup\mathcal{A}^{s-\alpha}v$ weakly in $\Theta^{\alpha}(\Omega)$ and $\mathcal{A}^{s-\alpha}v_n\to\mathcal{A}^{s-\alpha}v$ strongly in $C(\overline{\Omega})$. Choosing in \eqref{2.5} $(\varphi,\phi) = (\mathcal{A}^{s-\alpha}(v_n - v), 0)$, we obtain \[ \int_\Omega \mathcal{A}^{s-\alpha/2}(v_n - v)\mathcal{A}^{s-\alpha/2}v_n\,dx \leq \int_\Omega f(u_n)\mathcal{A}^{s-\alpha}(v_n - v)dx + \epsilon_n(\| \mathcal{A}^{s-\alpha}(v_n - v)\|_{\Theta^{\alpha}}). \] The first term on the right is estimated as \[ \| \mathcal{A}^{s-\alpha}(v_n - v)\|_C\int_\Omega| f(u_n)|\,dx\to 0, \] and thus one concludes again that \[ \int_\Omega|\mathcal{A}^{s-\alpha/2}v_n|^2\,dx \to\int_\Omega|\mathcal{A}^{s-\alpha/2}v|^2\,dx \] and hence also $v_n\to v$ strongly in $\Theta^{2s-\alpha}(\Omega)$. Thus, the conditions of Theorem \ref{teo2.4} are satisfied; hence, we find a positive critical point $(u, v)$ for the functional $\mathcal{J}$, which yields a weak solution to system \eqref{1}. \subsection{The case $p\leq 1$: Variational setting} Suppose that $p\leq 1$, $n=2$ and $s\in(1/2,1)$ or $n=3$ and $s\in(3/4,1)$. Thus, $\Theta^{2s}(\Omega)$ is compactly embedded in $C(\overline{\Omega})$. Let $\Omega$ be a smooth bounded open subset of $\mathbb{R}^n$ and $0 < s < 1$. To motivate our formulation, assume that the couple $(u,v)$ of nontrivial functions is roughly a solution of \eqref{1}. From the first equation, we have $v = (\mathcal{A}^{s} u)^{1/p}$. Plugging this equality into the second equation, we obtain \begin{equation} \label{S3.4} \begin{gathered} \mathcal{A}^{s} ( \mathcal{A}^{s} u )^{1/p} = f(u) \quad\text{in }\Omega \\ u = 0 \quad\text{on }\partial\Omega\,. \end{gathered} \end{equation} The basic idea in trying to solve \eqref{S3.4} is considering the functional $\Psi : \Theta^{2s}(\Omega) \to \mathbb{R}$ defined by \begin{equation}\label{func2} \Psi(u)=\frac{p}{p+1}\int_{\Omega}| \mathcal{A}^{s}u|^{\frac{p+1}{p}}dx -\int_{\Omega} F(u)dx\, . \end{equation} The Gateaux derivative of $\Psi$ at $u \in \Theta^{2s}(\Omega)$ in the direction $\varphi \in \Theta^{2s}(\Omega)$ is \[ \Psi'(u)\varphi=\int_{\Omega}|\mathcal{A}^{s}u|^{\frac{1}{p}-1} \mathcal{A}^{s}u \mathcal{A}^{s}\varphi\,dx - \int_{\Omega}f(u) \varphi\,dx \] and thus, if $f(u)\in C(\Omega)$, the problem \begin{gather*} \mathcal{A}^{s} v = f(u) \quad\text{in }\Omega\; \\ v = 0 \quad\text{on }\partial\Omega \end{gather*} admits a unique nontrivial weak solution $v \in \Theta^s(\Omega)$. Then, one easily checks that $u$ is a weak solution of the problem \begin{gather*} \mathcal{A}^{s} u = v^p \quad\text{in }\Omega\; \\ u = 0 \quad\text{on }\partial\Omega\,. \end{gather*} \begin{remark} \label{ponto} \rm In short, starting from a critical point $u \in \Theta^{2s}(\Omega)$ of $\Psi$, we have constructed a nontrivial weak solution $(u,v) \in \Theta^{2s}(\Omega) \times \Theta^s(\Omega)$ of the problem \eqref{1}. \end{remark} \subsection{Existence of critical points} In this subsection we prove the existence of a nontrivial weak solution of system \eqref{1}. By Remark \ref{ponto}, it suffices to show the existence of a nonzero critical point $u \in \Theta^{2s}(\Omega)$ of the functional $\Psi$. In this case $p\leq 1$, $n=2$ and $s\in\left(\frac{1}{2},1\right)$ or $n=3$ and $s\in\left(\frac{3}{4},1\right)$. Thus, $\Theta^{2s}(\Omega)$ is compactly embedded in $C(\overline{\Omega})$. Then, the second term of the functional $\Psi$ is defined if $F$ is continuous, and no growth restriction on $F$ is necessary. Since $F$ is differentiable, the functional $\Psi$ is a well-defined $C^1$-functional on the space $\Theta^{2s}(\Omega)$. The proof consists in applying the classical mountain pass theorem of Ambrosetti and Rabinowitz in our variational setting. We now show that $\Psi$ has a local minimum at the origin. \begin{align*} \Psi(u) &= \frac{p}{p+1}\int_{\Omega}| \mathcal{A}^{s}u|^{\frac{p+1}{p}}dx -\int_{\Omega} F(u)dx\\ &\geq \frac{p c}{p+1}\| u\|_{C}^{\frac{p+1}{p}} - o\big(\| u\|_{C}^{\frac{p+1}{p}}\big), \end{align*} so that the origin $u_0 = 0$ is a local minimum point. Next, let $u_1 = t \overline{u}$, where $t > 0$ and $\overline{u} \in \Theta^{2s}(\Omega)$ is a nonzero function. Then \[ \Psi(u_1)\leq\frac{p t^{\frac{p+1}{p}}}{p+1} \int_{\Omega} | \mathcal{A}^{s} \overline{u} |^{\frac{p+1}{p}}dx - t^{\theta} \| \overline{u}\|_{C}^{\theta}+d \] with $\theta>\frac{p+1}{p}$ (by assumption), and thus $\Psi(t\overline{u})\to-\infty$ as $t\to+\infty$. Finally, we show that $\Psi$ fulfills the Palais-Smale condition (PS). Let $(u_k) \subset \Theta^{2s}(\Omega)$ be a (PS)-sequence, that is, \begin{gather*} | \Psi(u_k)| \leq C_0, \\ | \Psi'(u_k) \varphi| \leq \epsilon_k \| \varphi \|_{\Theta^{2s}} \end{gather*} for all $\varphi \in \Theta^{2s}(\Omega)$, where $\epsilon_k \to 0$ as $k \to +\infty$. We have \begin{align*} &C_0 + \epsilon_k \| u_k \|_{\Theta^{2s}} \\ &\geq | \theta \Psi(u_k) - \Psi'(u_k) u_k| \\ &\geq \big(\theta\frac{p}{p+1}-1\big) \int_{\Omega}| \mathcal{A}^{s}u_k|^{\frac{p+1}{p}}dx -\theta\int_{\Omega}F(u_k)dx+\int_{\Omega}f(u_k)u_kdx\\ &\geq \big(\theta\frac{p}{p+1}-1\big) \int_{\Omega}| \mathcal{A}^{s}u_k|^{\frac{p+1}{p}}dx -C_0\geq \delta \| u_k\|_{\Theta^{2s}}^{\frac{p+1}{p}}-C_0, \end{align*} and thus $(u_k)$ is bounded in $\Theta^{2s}(\Omega)$. Thanks to the compactness of the embedding $\Theta^{2s}(\Omega) \hookrightarrow C(\overline{\Omega})$, one easily checks that $(u_k)$ converges strongly in $\Theta^{2s}(\Omega)$. So, by the mountain pass theorem, we obtain a nonzero critical point $u \in \Theta^{2s}(\Omega)$. This completes the proof. \section{Proof of theorem \ref{teo2}} Note that $n>4s$ implies that $\Theta^{2s}(\Omega)$ is continuously embedded in $L^{\frac{2n}{n-4s}}(\Omega)$. Now if $n=4s$ we have $\Theta^{2s}(\Omega)$ is compactly embedded in $L^{r}(\Omega)$ for all $r>1$. Thus if $u \in \Theta^{2s}(\Omega)$ we have that $u^q\in L^{\frac{2n}{n+2s}}(\Omega)$ for all $q>0$. It suffices to prove the result for $n > 4s$, since the ideas involved in its proof are fairly similar when $n = 4s$. \subsection{Variational setting} Let $\Omega$ be a smooth bounded open subset of $\mathbb{R}^n$ and $0 < s < 1$.To motivate our formulation, assume that the couple $(u,v)$ of nonnegative functions is roughly a solution of \eqref{2}. From the first equation, we have $v = \left(\mathcal{A}^{s} u\right)^{1/p}$. Plugging this equality into the second equation, we obtain \begin{equation} \label{S3.41} \begin{gathered} \mathcal{A}^{s} \left( \mathcal{A}^{s} u \right)^{1/p} = u^q \quad\text{in }\Omega\; \\ u \geq 0 \quad\text{in }\Omega \\ u = 0 \quad\text{on }\partial\Omega\,. \end{gathered} \end{equation} The basic idea for soving \eqref{S3.41} is considering the functional $\Phi : \Theta^{2s}(\Omega) \to \mathbb{R}$ defined by \begin{equation}\label{func3} \Phi(u)=\frac{p}{p+1}\int_{\Omega}| \mathcal{A}^{s}u|^{\frac{p+1}{p}}dx -\frac{1}{q+1}\int_{\Omega} (u^+)^{q+1}dx\, . \end{equation} The Gateaux derivative of $\Phi$ at $u \in \Theta^{2s}(\Omega)$ in the direction $\varphi \in \Theta^{2s}(\Omega)$ is \[ \Phi'(u)\varphi=\int_{\Omega}|\mathcal{A}^{s}u|^{\frac{1}{p}-1} \mathcal{A}^{s}u \mathcal{A}^{s}\varphi\,dx - \int_{\Omega}(u^+)^q \varphi\,dx\, . \] In this case, $\Theta^{2s}(\Omega)$ is continuously embedded in $L^{\frac{2n}{n-4s}}(\Omega)$. Thus, if $0 0$ such that \begin{align*} \Phi(u) &= \frac{p}{p+1}\int_{\Omega}| \mathcal{A}^{s}u|^{\frac{p+1}{p}}dx - \frac{1}{q+1}\int_{\Omega}| u| ^{q+1}dx\\ &\geq \frac{p C_1}{p+1}\| u\|_{\Theta^{2s}}^{\frac{p+1}{p}} -\frac{C_2}{q+1}\| u\|_{\Theta^{2s}}^{q+1}\\ &= \| u\|^{\frac{p+1}{p}}_{\Theta^{2s}} \Big(\frac{p C_1}{p+1}-\frac{C_2}{(q+1)\| u\|^{\frac{p+1}{p}-(q+1)}_{\Theta^{2s}}} \Big) \end{align*} for all $u\in \Theta^{2s}(\Omega)$. Therefore, $\Phi$ is lower bounded and coercive, that is, $\Phi(u) \to +\infty$ as $\| u\|_{\Theta^{2s}} \to +\infty$. Let $(u_k) \subset \Theta^{2s}(\Omega)$ be a minimizing sequence of $\Phi$. It is clear that $(u_k)$ is bounded in $\Theta^{2s}(\Omega)$, since $\Phi$ is coercive. So, module a subsequence, we have $u_k \rightharpoonup u_0$ in $\Theta^{2s}(\Omega)$. Since $\Theta^{2s}(\Omega)$ is compactly embedded in $L^{q+1}(\Omega)$, we have $u_k \to u_0$ in $L^{q+1}(\Omega)$. Here, again we use the fact that $q + 1 < \frac{p+1}{p}$. Thus \begin{align*} \lim_{n\to\infty}\inf \Phi(u_k) &= \lim_{k \to \infty} \inf \frac{p}{p+1}\| \mathcal{A}^{s} u_k \|^{\frac{p+1}{p}}_{L^{\frac{p+1}{p}}}-\frac{1}{q+1}\| u_{0}\|_{L^{q+1}}^{q+1}\\ &\geq \frac{p}{p+1}\| \mathcal{A}^{s}u_{0}\|^{\frac{p+1}{p}}_{L^{\frac{p+1}{p}}} -\frac{1}{q+1}\| u_{0}\|_{L^{q+1}}^{q+1} = \Phi(u_{0})\, , \end{align*} so that $u_0$ minimizers $\Phi$ on $\Theta^{2s}(\Omega)$. We just need to guarantee that $u_0$ is nonzero. But, this fact is clearly true since $\Phi(\epsilon u_1) < 0$ for any nonzero nonnegative function $u_1 \in \Theta^{2s}(\Omega)$ and $\epsilon > 0$ small enough; that is, \[ \Phi(\epsilon u_1) = \frac{p\epsilon^{\frac{p+1}{p}}}{p+1} \int_{\Omega}| \mathcal{A}^{s} u_1 |^{\frac{p+1}{p}}dx - \frac{\epsilon^{q+1}}{q+1}\int_{\Omega}| u_1 | ^{q+1}dx < 0 \] for $\epsilon>0$ small enough. This ends the proof of existence. \subsection{Uniqueness for the case $pq < 1$} The main tools in the proof of uniqueness are the strong maximum principle and a Hopf's lemma adapted to fractional operators. Let $(u_{1},v_{1}),(u_{2},v_{2})$ be two positive solutions of \eqref{2}. Define \[ \Gamma=\{\gamma\in(0,1] : u_{1} - tu_{2},\ v_{1} - tv_{2}\geq 0\ \text{ in } \overline{\Omega}\ \text{ for all }\ t \in [0,\gamma]\}\, . \] From the strong maximum principle and Hopf's lemma (see \cite{T}), it follows that $\Gamma$ is not empty. Let $\gamma_{\ast}=\sup \Gamma$ and assume that $\gamma_{\ast} < 1$. Clearly, \begin{equation}\label{eq1} u_{1}-\gamma_{\ast}u_{2},\ v_{1}-\gamma_{\ast}v_{2}\geq 0\quad \text{in } \overline{\Omega}\, . \end{equation} By \eqref{eq1} and the integral representation in terms of the Green function $G_{\Omega}$ of $\mathcal{A}^s$ (see \cite{Green, ros176}), we have \begin{align*} u_{1}(x)&= \int_{\Omega}G_{\Omega}(x,y)v_{1}^{p}(y)dy \\ &\geq\int_{\Omega}G_{\Omega}(x,y)\gamma_{\ast}^p v_{2}^{p}(y)dy\\ &= \gamma_{\ast}^{p}\int_{\Omega}G_{\Omega}(x,y)v_{2}^{p}(y)dy =\gamma_{\ast}^{p}u_{2}(x) \end{align*} for all $x \in \overline{\Omega}$. In a similar way, one gets $v_1 \geq \gamma_{\ast}^{q} v_2$ in $\overline{\Omega}$. Using the assumption $pq < 1$ and the fact that $\gamma_{\ast} < 1$, we derive \begin{equation} \begin{gathered} \mathcal{A}^s(u_{1} - \gamma_{\ast}u_{2}) = v_{1}^{p}-\gamma_{\ast}v_{2}^{p} \geq (\gamma_{\ast}^{pq}-\gamma_{\ast})v_{2}^{p} > 0 \\ \mathcal{A}^s(v_{1} - \gamma_{\ast}v_{2}) = u_{1}^{q}-\gamma_{\ast}u_{2}^{q} \geq (\gamma_{\ast}^{pq}-\gamma_{\ast})u_{2}^{q} > 0 \end{gathered} \end{equation} in $\Omega$ So, by the strong maximum principle, one has $u_{1}-\gamma_{\ast}u_{2}, v_{1}-\gamma_{\ast}v_{2} > 0$ in $\Omega$. Then, by Hopf's lemma, we have $\frac{\partial}{\partial\nu}(u_{1}-\gamma_{\ast}u_{2}), \frac{\partial}{\partial\nu}(v_{1}-\gamma_{\ast}v_{2})<0$ on $\partial \Omega$, where $\nu$ is the unit outer normal in $\mathbb{R}^n$ to $\partial\Omega$, so that $u_{1} - (\gamma_{\ast}+\epsilon)u_{2}, v_{1} - (\gamma_{\ast}+\epsilon)v_{2} > 0$ in $\Omega$ for $\epsilon > 0$ small enough, contradicting the definition of $\gamma_{\ast}$. Therefore, $\gamma_{\ast} \geq 1$ and, by \eqref{eq1}, $u_{1} - u_{2}, v_{1} - v_{2} \geq 0$ in $\overline{\Omega}$. A similar reasoning also produces $u_{2} - u_{1}, v_{2} - v_{1} \geq 0$ in $\overline{\Omega}$. This ends the proof of uniqueness. \subsection{Existence of critical points in the case $pq>1$} From what we saw, it suffices to show the existence of a nonzero critical point $u \in \Theta^{2s}(\Omega)$ of the functional $\Phi$. The proof consists in applying the classical mountain pass theorem of Ambrosetti and Rabinowitz in our variational setting. We first assert that $\Phi$ has a local minimum in the origin. Note that $p\leq 1$ and $q + 1 > \frac{p+1}{p}$ because $pq > 1$. Hence $q<\frac{n+4s}{n-4s}$ the embedding $\Theta^{2s}(\Omega) \hookrightarrow L^{q+1}(\Omega)$ is compact. Consider the set $\Gamma :=\left\{u\in \Theta^{2s}(\Omega) : \| u\|_{\Theta^{2s}} =\rho\right\}$. Then, on $\Gamma$, we have \begin{align*} \Phi(u) &= \frac{p}{p+1}\int_{\Omega}| \mathcal{A}^{s}u|^{\frac{p+1}{p}}dx -\frac{1}{q+1}\int_{\Omega}| u|^{q+1}dx\\ &\geq \frac{p C_1}{p+1}\| u\|_{\Theta^{2s}}^{\frac{p+1}{p}} - \frac{C_2}{q+1}\| u\|_{\Theta^{2s}}^{q+1} =\rho^{\frac{p+1}{p}}\Big( \frac{p C_1}{p+1} -\frac{C_2}{q+1}\rho^{q+1-\frac{p+1}{p}}\Big) \\ &> 0 = \Phi(0) \end{align*} for fixed $\rho>0$ small enough, so that the origin $u_0 = 0$ is a local minimum point. In particular, $\inf_{\Gamma} \Phi > 0 = \Phi(u_0)$. Note that $\Gamma$ is a closed subset of $\Theta^{2s}(\Omega)$ and decomposes $\Theta^{2s}(\Omega)$ into two connected components: $\left\{u\in \Theta^{2s}(\Omega) : \| u\|_{\Theta^{2s}} < \rho \right\}$ and $\left\{u\in \Theta^{2s}(\Omega) : \| u\|_{\Theta^{2s}} > \rho\right\}$. Let $u_1 = t \overline{u}$, where $t > 0$ and $\overline{u} \in \Theta^{2s}(\Omega)$ is a nonzero nonnegative function. Since $pq > 1$, we can choose $t$ sufficiently large so that \[ \Phi(u_1)=\frac{p t^{\frac{p+1}{p}}}{p+1} \int_{\Omega} | \mathcal{A}^{s} \overline{u} |^{\frac{p+1}{p}}dx - \frac{t^{q+1}}{q+1} \int_{\Omega} (\overline{u}^+)^{q+1}dx < 0\, . \] It is clear that $u_1 \in \left\{u\in \Theta^{2s}(\Omega): \| u\|_{\Theta^{2s}} > \rho\right\}$ and $\inf_{\Gamma} \Phi > \max\{\Phi(u_0), \Phi(u_1)\}$, so that the mountain pass geometry is satisfied. Finally, we show that $\Phi$ fulfills the Palais-Smale condition (PS). Let $(u_k) \subset \Theta^{2s}(\Omega)$ be a (PS)-sequence; that is, \begin{gather*} | \Phi(u_k)| \leq C_0, \\ | \Phi'(u_k) \varphi| \leq \epsilon_k \| \varphi \|_{\Theta^{2s}} \end{gather*} for all $\varphi \in \Theta^{2s}(\Omega)$, where $\epsilon_k \to 0$ as $k \to +\infty$. From these two inequalities and the assumption $pq > 1$, we deduce that \begin{align*} C_0 + \epsilon_k \| u_k \|_{\Theta^{2s}} &\geq | (q+1) \Phi(u_k) - \Phi'(u_k) u_k| \\ &\geq \Big(\frac{p(q+1)}{p+1}-1\Big) \int_{\Omega}| \mathcal{A}^{s}u_k|^{\frac{p+1}{p}}dx\\ &\geq C \| u_k\|_{\Theta^{2s}}^{\frac{p+1}{p}} \end{align*} and thus $(u_k)$ is bounded in $\Theta^{2s}(\Omega)$. Thanks to the compactness of the embedding $\Theta^{2s}(\Omega) \hookrightarrow L^{q+1}(\Omega)$, one easily checks that $(u_k)$ converges strongly in $\Theta^{2s}(\Omega)$. So, by the mountain pass theorem, we obtain a nonzero critical point $u \in \Theta^{2s}(\Omega)$. This completes the proof. \section{Regularity of solutions to systems \ref{1} and \ref{2}} In this section, we establish regularity property of the weak solutions of system \eqref{1} based on the results obtained in \cite{T}. Also we establish the Brezis-Kato type result and derive the regularity of solutions to \eqref{2}. \begin{proposition} \label{prop5.1} Let $(u,v)$ be a weak solution of the problem \eqref{1}. In the hypothesis of Theorem \ref{teo1}, we have $(u, v) \in L^{\infty}(\Omega) \times L^{\infty}(\Omega)$ and, moreover, $(u,v) \in C^{1,\beta}(\overline{\Omega}) \times C^{\beta}(\overline{\Omega})$ for some $\beta \in (0, 1)$. Now if $f$ is a $C^1$ function such that $f(0)=0$ we have $(u,v) \in C^{1,\beta}(\overline{\Omega}) \times C^{1,\beta}(\overline{\Omega})$ for some $\beta \in (0, 1)$. \end{proposition} \begin{proof} In the case $p>1$ we find a solution $(u,v)\in\Theta^{\alpha}(\Omega) \times \Theta^{2s-\alpha}(\Omega)$. By choosing $\alpha$ (see Lemma \ref{lema3.1}), and by Sobolev imbedding theorem (see \cite{DD}) we have $u\in L^{\infty}(\Omega)$. Then $f(u)\in L^{\infty}(\Omega)$. Thus, by regularity result (see \cite[Proposition 3.1]{T}) we have $v\in C^{\gamma}(\overline{\Omega})$ for some $\gamma \in (0, 1)$. Hence $\gamma +2s>1$ and $v^p\in C^{\gamma}(\overline{\Omega})$ again by \cite[Proposition 3.1]{T} we have $u\in C^{1,\gamma+2s-1}(\overline{\Omega})$. Therefore $(u,v) \in C^{1,\beta}(\overline{\Omega}) \times C^{\beta}(\overline{\Omega})$ for some $\beta \in (0, 1)$. Now if $f$ is a $C^1$ function such that $f(0)=0$ analogously we have $(u,v) \in C^{1,\gamma+2s-1}(\overline{\Omega}) \times C^{\gamma}(\overline{\Omega})$ for some $\gamma \in (0, 1)$. Then $f(u)\in C^{2s}(\overline{\Omega})$. Hence $2s +2s>1$ again by \cite[Proposition 3.1]{T} we have $v\in C^{1,2s+2s-1}(\overline{\Omega})$. Therefore $(u,v) \in C^{1,\beta}(\overline{\Omega}) \times C^{1,\beta}(\overline{\Omega})$ for some $\beta \in (0, 1)$. In the case $p\leq 1$ we find a solution $(u,v)\in\Theta^{2s}(\Omega) \times \Theta^{s}(\Omega)$. From Sobolev imbedding theorem (see \cite{DD}) we have $u\in L^{\infty}(\Omega)$. Analogous to the previous case, we have the result. \end{proof} Next we prove the $L^\infty$ estimate of Brezis-Kato type. \begin{proposition} \label{prop5.2} Let $(u,v)$ be a weak solution of the problem \eqref{2}. In the hypothesis of Theorem \ref{teo2}, we have $(u, v) \in L^{\infty}(\Omega) \times L^{\infty}(\Omega)$ and, moreover, $(u,v) \in C^{1,\beta}(\overline{\Omega}) \times C^{1,\beta}(\overline{\Omega})$ for some $\beta \in (0, 1)$. \end{proposition} \begin{proof} In the case $n>4s$ we find a solution \[ (u,v)\in \begin{cases} \Theta^{2s}(\Omega) \times \Theta^s(\Omega) &\text{if }01$, we rewrite the problem \eqref{2} as follows \begin{equation} \begin{gathered} \mathcal{A}^{s}u = a(x)v^{p/2} \quad\text{in }\Omega\\ \mathcal{A}^{s}v = b(x)u \quad\text{in }\Omega\\ u= v=0 \quad\text{in }\mathbb{R}^n\setminus\Omega\,, \end{gathered} \end{equation} where $a(x) = v(x)^{p/2}$ and $b(x) = u(x)^{q - 1}$. Note that $p+1<\frac{2n}{n-2s}$ and $p+1\leq 2<\frac{2n}{n-2r}$. By Sobolev embedding, $\Theta^{s}(\Omega)\hookrightarrow L^{p+1}(\Omega)$ and $\Theta^{r}(\Omega)\hookrightarrow L^{p+1}(\Omega)$, so that $a \in L^{\frac{2(p+1)}{p}}(\Omega)$. Thus, for each fixed $\epsilon > 0$, we can construct functions $q_{\epsilon} \in L^{\frac{2(p+1)}{p}}(\Omega)$, $f_{\epsilon} \in L^{\infty}(\Omega)$ and a constant $K_{\epsilon} > 0$ such that \begin{gather*} a(x) v(x)^{p/2} = q_{\epsilon}(x)v(x)^{p/2} + f_{\epsilon}(x),\\ \| q_{\epsilon} \|_{L^{\frac{2(p+1)}{p}}} < \epsilon,\quad \| f_{\epsilon} \|_{L^{\infty}} < K_{\epsilon}\, . \end{gather*} In fact, consider the set $\Omega_k = \{x \in\Omega: | a(x)| < k\}$, where $k$ is chosen such that \[ \int_{\Omega_k^c} | a(x)|^{\frac{2(p+1)}{p}}dx < \frac{1}{2}\epsilon^{\frac{2(p+1)}{p}}\, . \] This condition is clearly satisfied for $k = k_\epsilon$ large enough. We now write \begin{equation} q_{\epsilon}(x)=\begin{cases} \frac{1}{m}a(x) &\text{for } x \in \Omega_{k_\epsilon}\\ a(x) &\text{for } x \in \Omega_{k_\epsilon}^c \end{cases} \end{equation} and $f_{\epsilon}(x) = \left(a(x) - q_{\epsilon}(x)\right)v(x)^{p/2}$. Then \begin{align*} \int_{\Omega}| q_{\epsilon}(x)|^{\frac{2(p+1)}{p}}dx &= \int_{\Omega_{k_\epsilon}}| q_{\epsilon}(x)|^{\frac{2(p+1)}{p}}dx +\int_{\Omega_{k_\epsilon}^c}| q_{\epsilon}(x)|^{\frac{2(p+1)}{p}}dx\\ &= \big(\frac{1}{m}\big)^{\frac{2(p+1)}{p}} \int_{\Omega_{k_\epsilon}}| a(x)|^{\frac{2(p+1)}{p}}dx +\int_{\Omega_{k_\epsilon}^c}| a(x)|^{\frac{2(p+1)}{p}}dx\\ &< \big(\frac{1}{m}\big)^{\frac{2(p+1)}{p}} \int_{\Omega_{k_\epsilon}}| a(x)|^{\frac{2(p+1)}{p}}dx +\frac{1}{2}\epsilon^{\frac{2(p+1)}{p}}\, . \end{align*} So, for $m = m_\epsilon > \big(\frac{2^{\frac{p}{2(p+1)}}}{\epsilon}\big) \| a\|_{L^{\frac{2(p+1)}{p}}}$, we obtain \[ \| q_{\epsilon}\|_{L^{\frac{2(p+1)}{p}}} < \epsilon\, . \] Note also that $f_{\epsilon}(x) = 0$ for all $x \in \Omega_{k_\epsilon}^c$ and, for this choice of $m$, \[ f_{\epsilon}(x) = \Big( 1-\frac{1}{m_\epsilon}\Big) a(x)^2 \leq \Big( 1-\frac{1}{m_\epsilon}\Big) k_\epsilon^2 \] for all $x \in \Omega_{k_\epsilon}$. Therefore, \[ \| f_{\epsilon} \|_{L^{\infty}} \leq \Big( 1-\frac{1}{m_\epsilon}\Big) k_\epsilon^2 := K_\epsilon\, . \] On the other hand, $v(x) = \mathcal{A}^{-s}(bu)(x)$, where $b\in L^{\frac{q+1}{q-1}}(\Omega)$. Hence, \[ u(x) = \mathcal{A}^{-s}[q_{\epsilon}(x)(\mathcal{A}^{-s}(bu)(x))^{p/2}] + \mathcal{A}^{-s}f_{\epsilon}(x)\, . \] By \cite[Lemma 2.1]{choi}, the claims (ii) and (iv) below follow readily and, by using H\"{o}lder's inequality, we also get the claims (i) and (iii). Precisely, for fixed $\gamma > 1$, we have: \begin{itemize} \item[(i)] The map $w \to b(x)w$ is bounded from $L^\gamma(\Omega)$ to $L^\beta(\Omega)$ for \[ \frac{1}{\beta} = \frac{q-1}{q+1} + \frac{1}{\gamma}; \] \item[(ii)] For $2s=n\big(\frac{1}{\beta}-\frac{2}{p \theta}\big)$, there exists a constant $C > 0$, depending on $\beta$ and $\theta$, such that \[ \| (\mathcal{A}^{s}w)^{p/2}\|_{L^\theta}\leq C\| w\|_{L^\beta}^{p/2} \] for all $w \in L^\beta(\Omega)$; \item[(iii)] The map $ w \to q_\epsilon(x)w$ is bounded from $L^\theta(\Omega)$ to $L^\eta(\Omega)$ with norm given by $\| q_\epsilon \|_{L^{\frac{2(p+1)}{p}}}$, where $\theta \geq 1$ and $\eta$ satisfies \[ \frac{1}{\eta}=\frac{p}{2(p+1)}+\frac{1}{\theta}; \] \item[(iv)] For $2s = n\big(\frac{1}{\eta} - \frac{1}{\delta}\big)$, the map $ w\to \mathcal{A}^{-s}w$ is bounded from $L^\eta(\Omega)$ to $L^\delta(\Omega)$. \end{itemize} From (i)--(iv), one easily checks that $\gamma < \delta$ and, in addition, \begin{align*} \| u\|_{L^\delta} &\leq \| \mathcal{A}^{-s}\big[q_\epsilon(x)\left(\mathcal{A}^{-s}(bu)\right)^{p/2} \big]\|_{L^\delta}+\| \mathcal{A}^{-s}f_\epsilon\|_{L^\delta}\\ &\leq C \Big(\| q_\epsilon\|_{L^{\frac{2(p+1)}{p}}}\| u\|_{L^\delta}^{p/2} + \| f_\epsilon\|_{L^\delta}\Big). \end{align*} Using now the fact that $p \leq 1$, $\| q_\epsilon \|_{L^{\frac{2(p+1)}{p}}} < \epsilon$ and $f_\epsilon \in L^\infty(\Omega)$, we deduce that $\| u \|_{L^\delta} \leq C$ for some constant $C > 0$ independent of $u$. Proceeding inductively, we obtain $u \in L^\delta(\Omega)$ for all $\delta \geq 1$. So, \cite[Lemma 2.1]{choi} implies that $v \in L^{\infty}(\Omega)$. From this, and using \cite[Lemma 2.1]{choi} again, we deduce that $u \in L^{\infty}(\Omega)$. Then $u^q,v^p\in L^{\infty}(\Omega)$. Thus, by regularity result (see \cite[Proposition 3.1]{T}) we have $v,u\in C^{2s}(\overline{\Omega})$. Hence it holds that $u^q,v^p \in C^{2s}(\overline{\Omega})$. Again, we can apply regularity result to deduce that $u\in C^{4s}(\overline{\Omega})$. Iteratively, we can raise the regularity so that $(u,v) \in C^{1,\beta}(\overline{\Omega}) \times C^{1,\beta}(\overline{\Omega})$ for some $\beta \in (0, 1)$. Now if $q \leq 1$ write $b(x) = u(x)^{\frac{q}{2}}$. In case $n=4s$ we find a solution $(u,v)\in \Theta^{2s}(\Omega) \times \Theta^s(\Omega)$. Since $\Theta^{2s}(\Omega)$ is compactly embedded in $L^{r}(\Omega)$ for all $r>1$, the result follows similarly. \end{proof} \begin{thebibliography}{99} \bibitem{A} {D. Applebaum}; \emph{L\'{e}vy processes -- from probability to finance and quantum groups}, Notices Amer. Math. Soc., 51 (2004), 1336-1347. \bibitem{BaCPS} B. Barrios, E. Colorado, A. de Pablo, U. S\'{a}nchez; \emph{On some critical problems for the fractional Laplacian operator}, J. Differential Equations, 252 (2012), 6133-6162. \bibitem{Green} M. Bonforte, J. L. V\'{a}zquez; \emph{A priori estimates for fractional nonlinear degenerate diffusion equations on bounded domains}, Arch. Ration. Mech. Anal., 218 (2015), 317-362. \bibitem{BrCPS} C. Br\"{a}ndle, E. Colorado, A. de Pablo, U. S\'{a}nchez; \emph{A concave-convex elliptic problem involving the fractional Laplacian}, Proceedings of the Royal Society of Edinburgh, 143A (2013), 39-71. \bibitem{bucur} C. Bucur, E. Valdinoci; \emph{Nonlocal diffusion and applications}, volume 20 of Lecture Notes of the Unione Matematica Italiana. Springer, [Cham]; Unione Matematica Italiana, Bologna, 2016. xii+155 pp. ISBN: 978-3-319-28738-6; 978-3-319-28739-3. \bibitem{CT} X. Cabr\'{e}, J. Tan; \emph{Positive solutions of nonlinear problems involving the square root of the Laplacian}, Advances in Mathematics, 224 (2010), 2052-2093. \bibitem{CDDS} A. Capella, J. D\'{a}vila, L. Dupaigne, Y. Sire; \emph{Regularity of Radial Extremal Solutions for Some Non-Local Semilinear Equations}, Comm. Partial Differential Equations, 36 (2011), 1353-1384. \bibitem{choi} W. Choi; \emph{On strongly indefinite systems involving the fractional Laplacian}, Nonlinear Anal., 120 (2015), 127-153. \bibitem{CK} W. Choi, S. Kim, K. Lee; \emph{Asymptotic behavior of solutions for nonlinear elliptic problems with the fractional Laplacian}, J. Funct. Anal., 266 (2014), 6531-6598. \bibitem{CFM} Ph. Cl\'{e}ment, D. G. de Figueiredo, E. Mitidieri; \emph{Positive solutions of semilinear elliptic systems}, Comm. Partial Differential Equations, 17 (1992), 923-940. \bibitem{torus} R. de la Llave, E. Valdinoci; \emph{A generalization of Aubry-Mather theory to partial differential equations and pseudo-differential equations}, Ann. Inst. H. Poincar\'{e} Anal. Non Lin\'{e}aire 26 (2009), no. 4, 1309-1344. \bibitem{DD} F. Demengel, G. Demengel; \emph{Functional Spaces for the Theory of Elliptic Partial Differential Equations}, (Universitext)-Springer (2012). \bibitem{serena5} S. Dipierro, N. Soave, E. Valdinoci; \emph{On stable solutions of boundary reaction-diffusion equations and applications to nonlocal problems with Neumann data}, Indiana Univ. Math. J., https: //www.iumj.indiana.edu/IUMJ/Preprints/6282.pdf \bibitem{FM} P. Felmer, S. Mart\'{i}nez; \emph{Existence and uniqueness of positive solutions to certain differential systems}, Adv. Differential Equations, 4 (1998), 575-593. \bibitem{DG} D. G. de Figueiredo; \emph{Semilinear elliptic systems}, Nonl. Funct. Anal. Appl. Diff. Eq., World Sci. Publishing, River Edge (1998), 122-152. \bibitem{DF} D.G. de Figueiredo, P. Felmer; \emph{On superquadratic elliptic systems}, Trans. Amer. Math. Soc. 343 (1994), 99-116. \bibitem{DR} D. G. Figueiredo, B. Ruf; \emph{Elliptic systems with nonlinearities of arbitrary growth}, Mediterr. J. Math., 1 (2004), 417-431. \bibitem{vander} J. Hulshof, R. van der Vorst; \emph{Differential Systems with Strongly Indefinite Variational Structure}, J. Funct. Anal., 114 (1993), 32-58. \bibitem{ros176} T. Jakubowski; \emph{The estimates for the Green function in Lipschitz domains for the symmetric stable processes}, Probab. Math. Statist. 22 (2002), 419-441. \bibitem{E2} E. Leite; \emph{On strongly indefinite systems involving fractional elliptic operators}, arXiv: 1706.01349, 2017. To appear in Far East Journal of Applied Mathematics. \bibitem{EM} E. Leite, M. Montenegro; \emph{A priori bounds and positive solutions for non-variational fractional elliptic systems}, arXiv: 1409.6060, 2014. To appear in Differential and Integral Equations. \bibitem{EM1} E. J. F. Leite, M. Montenegro; \emph{On positive viscosity solutions of fractional Lane-Emden systems}, arXiv: 1509.01267, 2015. \bibitem{LI} S. Li, M. Willem; \emph{Applications of local linking to critical point theory}, J. Math. Anal. Appl., 189 (1995), 6-32. \bibitem{Mi1} {E. Mitidieri}; \emph{A Rellich type identity and applications}, Comm. Partial Differential Equations, 18 (1993), 125-151. \bibitem{Mi2} E. Mitidieri; \emph{Nonexistence of positive solutions of semilinear elliptic systems in $\mathbb{R}^N$}, Differential and Integral Equations, 9 (1996), 465-479. \bibitem{marcos} M. Montenegro; \emph{The construction of principal spectral curves for Lane-Emden systems and applications}, Ann. Scuola Norm. Sup. Pisa Cl. Sci., 29 (2000), 193-229. \bibitem{compare} R. Musina, A. I. Nazarov; \emph{On fractional Laplacians}, Comm. Partial Differential Equations, 39 (2014), 1780-1790. \bibitem{SZ1} J. Serrin, H. Zou; \emph{Existence of positive entire solutions of elliptic Hamiltonian systems}, Comm. Partial Differential Equations, 23 (1998), 577-599. \bibitem{rafaella} R. Servadei, E. Valdinoci; \emph{On the spectrum of two different fractional operators}, Proc. Roy. Soc. Edinburgh, Sect. A 144 (2014), 831-855. \bibitem{tan1} J. Tan; \emph{The Brezis-Nirenberg type problem involving the square root of the Laplacian}, Calc. Var. Partial Differential Equations, 42 (2011), 21-41. \bibitem{T} J. Tan; \emph{Positive solutions for non local elliptic problems}, Discrete and Continuous Dynamical Systems, 33 (2013), 837-859. \bibitem{yang} A. Xia, J. Yang; \emph{Regularity of nonlinear equations for fractional Laplacian}, Proc. Amer. Math. Soc., 141 (2013), 2665-2672. \end{thebibliography} \end{document}