\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2013 (2013), No. 33, pp. 1--10.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2013 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2013/33\hfil Positive blowup solutions] {Positive blowup solutions for some fractional systems in bounded domains} \author[R. Alsaedi \hfil EJDE-2013/33\hfilneg] {Ramzi Alsaedi} % in alphabetical order \address{Ramzi Alsaedi \newline Department of Mathematics, College of Science and Arts, King Abdulaziz University, Rabigh Campus, P. O. Box 344, Rabigh 21911, Saudi Arabia} \email{ramzialsaedi@yahoo.co.uk} \thanks{Submitted November 2, 2012. Published January 30, 2013.} \subjclass[2000]{26A33, 34B27, 35B44, 35B09} \keywords{Fractional nonlinear systems, Green function, positive solutions,\hfill\break\indent maximum principle} \begin{abstract} Using some potential theory tools and the Schauder fixed point theorem, we prove the existence of a positive continuous weak solution for the fractional system $$ ( -\Delta )^{\alpha/2}u+ p(x)u^{\sigma }v^{r}=0,\quad (-\Delta)^{\alpha/2}v+q(x)u^{s}v^{\beta }=0 $$ in a bounded $ C^{1,1}$-domain $D$ in $\mathbb{R}^{n}$ $(n\geq 3)$, subject to Dirichlet conditions, where $0<\alpha <2$, $\sigma ,\beta \geq 1$, $s,r\geq 0$. The potential functions $p,q$ are nonnegative and required to satisfy some adequate hypotheses related to the Kato class $K_{\alpha }(D)$. We also investigate the global behavior of such solution. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{lemma}[theorem]{Lemma} \newtheorem{proposition}[theorem]{Proposition} \newtheorem{definition}[theorem]{Definition} \allowdisplaybreaks \section{Introduction and statement of main results} Let $D$ be a bounded $C^{1,1}$-domain in $\mathbb{R}^{n}$, $(n\geq 3)$ and $0<\alpha<2$. This paper is devoted to the study of the following system involving the fractional Laplacian \begin{equation}\label{S1} \begin{gathered} (-\Delta )^{\alpha/2}u+p(x)u^{\sigma }v^{r}=0 \quad \text{in }D, \\ (-\Delta )^{\alpha/2}v+q(x)u^{s}v^{\beta }=0 \quad \text{in }D, \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_D^{\alpha }1(x)}=\varphi (z), \\ \lim_{x\to z\in \partial D} \frac{v(x)}{M_D^{\alpha }1(x)}=\psi (z), \end{gathered} \end{equation} where $\sigma ,\beta \geq 1$, $r,s\geq 0$ , the functions $\varphi $ and $\psi $ are positive continuous on $\partial D$ and the nonnegative potential functions $p,q$ are required to satisfy some adequate hypotheses related to the Kato class $K_{\alpha }(D)$ (see Definition $\ref{def1.1}$ below). The function ${M_D^{\alpha }1(x)}$ is defined on $D$ by \begin{equation} \label{MDalpha1} M_D^{\alpha }1(x)=\int_{\partial D}M_D^{\alpha}(x,z)\nu(dz). \end{equation} Here, $\nu$ is an appropriate measure on $\partial D$ which will be defined later in \eqref{def-nu} and $M_D^{\alpha}(x,z)$ is the Martin kernel of the killed symmetric $\alpha$-stable process $X^D=(X_t^D)_{t>0}$ in $D$ associated to $(-\Delta)^{\alpha/2}$. For the reader convenience, we recall the definition of the fractional Laplacian $- {( -\Delta )^{\alpha/2}}$ which is a nonlocal operator and can be defined by the formula \[ -(-\Delta )^{\alpha/2}u(x) = c_{n\,,\,\alpha} \lim_{\epsilon \searrow 0}\int_{(|x-y|>\epsilon)} \frac{u(y)-u(x)}{|x-y|^{n+\alpha}}, \] where $c_{n,\alpha}$ is a dimensional constant that depends on $n$ and $\alpha$ (see \cite{BB,B,DPV} for more details). Fractional Laplacian is of interest in many branches of sciences such as physics, biologists, queuing theory, operation research, mathematical finance and risk estimation. The fractional powers of the Laplacian in all of ${\mathbb{R}}^n$ are useful to describe anomalous diffusions in plasmas, flames propagation and chemical reactions in liquids, population dynamics, geophysical fluid dynamics, and American options in finance, see \cite{Ap,JW,KSZ}. In the classical case (i.e. $\alpha =2$), there is a large amount of literature dealing with the existence, nonexistence and qualitative analysis of positive solutions for problems related to \eqref{S1}; see for example, the papers of Cirstea and Radulescu \cite{CR}, Ghanmi et al \cite{GMTZ}, Ghergu and Radulescu \cite{GR}, Lair and Wood \cite{LW1}, \cite{LW2}, Mu et al \cite{MHTL} and references therein. In these works various existence results of positive bounded solutions or positive blow-up solutions (called also large solutions) have been established and a precise global behavior is given. We note also that several methods have been used to treat these systems such as sub and super-solutions method, variational method and topological methods. These results have been extended recently by Alsaedi et al in \cite{AMZ} for $n\geq 3$ and by Alsaedi in \cite{Als} for $n=2$, in the case $\alpha=2$, $\sigma,\beta \geq 1$, $s>0$, $r>0$, where the authors established the existence of a positive continuous bounded solution for \eqref{S1} in the case $n\geq 3$ and positive continuous solution having logarithm growth at infinity in an exterior domain of ${\mathbb{R}}^2$. Recently, there has been intensive interest in studying the fractional Laplacian $(-\Delta)^{\alpha/2}$, the development of its potential theory and the global behavior of its Green function $G_D^{\alpha}$, see \cite{CS1,CS2,CS3,CK}. In this article, we will exploit these potential theory tools and the properties of the Kato class $K_{\alpha}(D)$, defined and studied in \cite{CMM}, to study the existence of positive continuous solutions (in the sense of distributions) for \eqref{S1}. More precisely, we aim first at proving the existence and uniqueness of a positive continuous solution (in the sense of distributions) for the scalar equation \begin{equation} \begin{gathered} (-\Delta )^{\alpha/2}u+p_0(x)u^{\gamma }=0 \quad \text{in }D, \\ u>0 \quad \text{in }D, \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_D^{\alpha }1(x)}=\varphi (z), \end{gathered} \label{e1.3} \end{equation} where $\gamma \geq 1$ and $p_0$ is a nonnegative Borel measurable function in $D$ satisfying the condition \begin{itemize} \item[(H1)] The function $x\to (\delta (x))^{(\frac{\alpha}{2} -1)(\gamma -1) }p_0(x)\in K_{\alpha }(D)$, \end{itemize} where $\delta (x)$ denotes the Euclidian distance from $x$ to the boundary of $D$ and the class $K_{\alpha}(D)$ is defined by means of the Green{'}s function $G_D^{\alpha}$ of $(-\Delta)^{\alpha/2}$ as follows. \begin{definition}[\cite{CMM}] \label{def1.1} \rm A Borel measurable function $\varphi$ in $D$ belongs to the Kato class $K_{\alpha }(D)$ if \[ \lim_{r\to 0}\Big(\sup_{x\in D}\int\nolimits_{(| x-y| \leq r)\cap D}\big(\frac{\delta (y)}{\delta (x)} \big)^{\alpha/2} G_D^{\alpha}(x,y)|\varphi(y)|dy\Big)=0. \] \end{definition} It has been shown in \cite{CMM}, that the function \begin{equation} x\to (\delta (x))^{-\lambda } \text{ belongs } K_{\alpha }(D) \text{ if and only if } \lambda <\alpha . \label{e1.4} \end{equation} For more examples of functions belonging to $K_{\alpha }(D)$, we refer to \cite{CMM}. Note that for the classical case (i.e. $\alpha =2$), the class $K_2(D)$ was introduced and studied in \cite{MZ}. Using \eqref{e1.4}, hypothesis (H1) is satisfied if $p_0$ verifies the following condition: There exists a constant $C>0$, such that for each $x\in D$, \[ p_0(x)\leq \frac{C}{(\delta (x))^{\tau }},\quad \text{with }\tau +(1-\frac{\alpha}{2} )(\gamma -1)<\alpha . \] To state our existence result for \eqref{S1}, we denote by $M_D^{\alpha} \varphi$ (see \cite{CMM}), the unique positive continuous solution of \begin{equation} \begin{gathered} (-\Delta )^{\alpha/2}u=0 \quad \text{in }D,\; (\text{in the sense of distributions}) \\ \lim_{x\to z\in \partial D} \frac{u(x)}{M_D^{\alpha }1(x)}=\varphi (z)\,. \end{gathered} \label{e1.5} \end{equation} We recall also that in \cite{CS2}, the authors have proved the existence of a constant $C>0$ such that for each $x\in D$, \begin{equation}\label{e1.2} \frac{1}{C}(\delta (x))^{\frac{\alpha}{2} -1}\leq M_D^{\alpha }1(x)\leq C(\delta (x))^{\frac{\alpha}{2} -1}. \end{equation} Using some potential theory tools and an approximating sequence, we establish the following result. \begin{theorem} \label{thm1.2} Under hypothesis {\rm (H1)}, problem \eqref{e1.3} has a unique positive continuous solution satisfying for each $x\in D$ \[ c_0M_D^{\alpha }\varphi (x)\leq u(x)\leq M_D^{\alpha }\varphi (x), \] where the constant $c_0\in (0,1]$. \end{theorem} Next we exploit the result of Theorem \ref{thm1.2} to prove the existence of a positive continuous solution $(u,v)$ to the system $(\eqref {S1}$. To this end, we assume the following hypothesis: \begin{itemize} \item[(H2)] The functions $p,q$ are nonnegative Borel measurable functions such that \[ x\mapsto (\delta (x))^{(\frac{\alpha}{2} -1)( \sigma +r-1)}p(x)\in K_{\alpha }(D), \quad x\mapsto (\delta (x))^{(\frac{\alpha}{2} -1)(\beta +s-1) }q(x)\in K_{\alpha }(D). \] \end{itemize} Then by using the Schauder's fixed point theorem, we prove the following result. \begin{theorem} \label{thm1.3} Under assumption {\rm (H2)}, system \eqref{S1} has a positive continuous solution $(u,v)$ satisfying: for each $x\in D$, \[ c_1M_D^{\alpha }\varphi (x)\leq u(x)\leq M_D^{\alpha }\varphi (x) \text{ and }c_2M_D^{\alpha }\psi (x)\leq v(x)\leq M_D^{\alpha }\psi (x), \] where $c_1, c_2$ constants in $(0,1]$. \end{theorem} We note that contrary to the classical case $\alpha =2$ and $n\geq 3$, in our situation the solution blows up on the boundary of $D$. The content of this paper is organized as follows. In Section 2, we collect some properties of functions belonging to the Kato class $K_{\alpha }(D)$, which are useful to establish our results. Our main results are proved in Section 3. As usual, let $B^{+}(D)$ be the set of nonnegative Borel measurable functions in $D$. We denote by $C_0(D)$ the set of continuous functions in $D$ vanishing continuously on $\partial D$. Note that $C_0(D)$ is a Banach space with respect to the uniform norm $\| u\|_{\infty }=\underset{x\in D}{\sup }| u(x)| $. When two positive functions $f$ and $g$ are defined on a set $S$, we write $f\approx g $ if the two sided inequality $\frac{1}{C}g\leq f\leq Cg$ holds on $S$. Let $G_D$ be the Green function of the Dirichlet Laplacian in $D$. The Martin kernel $M_D(.,.)$ of the killed Brownian motion is defined by \[ M_D(x,z)=\lim_{D \ni y\to z}\frac{G_D(x,y)}{G_D(x,z)}\quad \text{for $x\in D$ and } z \in \partial D. \] Similarly, the Martin Kernel of the killed process $X^D$ is defined by \[ M_D^{\alpha}(x,z)=\lim_{D \ni y\to z}\frac{G_D^{\alpha}(x,y)}{G_D^{\alpha}(x,z)} \quad \text{for } x\in D \text{ and } z \in \partial D. \] Using, the Hergoltz theorem, there exists a positive measure $\nu$ in $ \partial D$ such that \begin{eqnarray}\label{def-nu} 1=\int_{\partial D} M_D(x,z)\, \nu(dz). \end{eqnarray} This measure $\nu$ is used in \eqref{MDalpha1} to define $M_D^{\alpha}1$. We define the potential kernel $G_D^{\alpha }$ of $X^{D}$ by \begin{equation} G_D^{\alpha }f(x):=\int_{D}G_D^{\alpha }(x,y)f(y) dy,\text{ for }f\in B^{+}(D)\text{ and }x\in D. \label{e1.6} \end{equation} Finally, let us recall some potential theory tools that will be needed in section 3 and we refer to \cite{CMM,CZ,M} for more details. For $q\in B^{+}(D)$, we define the kernel $V_{q}$ on $B^{+}(D)$ by \begin{equation} V_{q}f(x):=\int_0^{\infty }E^{x}( e^{-\int_0^{t}q(X_s^{D})ds}f(X_t^{D}))dt, \quad x\in D, \label{e1.7} \end{equation} with $V_0:=V=G_D^{\alpha }$ and $E^x$ stands for the expectation with respect to the symmetric $\alpha$-stable process $X^D$ starting from $x$. If $q$ satisfies $Vq<\infty$, we have the following resolvent equation \begin{equation}\label{e1.8} V=V_{q}+V_{q}(qV)=V_{q}+V(qV_{q}). \end{equation} It follows that for each each measurable function $u$ in $D$ such that $V( q|u|) <\infty$, we have \begin{equation}\label{e1.9} (I-V_{q}(q.))(I+V(q.))u=(I+V(q.))(I-V_{q}(q.))u=u. \end{equation} \section{The Kato class $K_{\protect\alpha }(D)$} \begin{proposition} \label{prop2.1}[\cite{CMM}] Let $q$ be a function in $K_{\alpha}(D)$, then we have \begin{itemize} \item[(i)] $a_{\alpha }(q):=\sup_{{x,y\in D}} \int_{D}\frac{G_D^{\alpha }(x,z)G_D^{\alpha }(z,y)}{G_D^{\alpha}(x,y)} | q(z)| dz<\infty$. \item[(ii)] Let $h$ be a positive $\alpha$-superharmonic function with respect to $X^D$. Then, for all $x \in D$ we have \begin{equation}\label{e2.2} \int_{D}G_D^{\alpha }(x,y)h(y)|q(y)|dy\leq a_{\alpha }(q)h(x). \end{equation} Furthermore, for each $x_0\in \overline{D}$, we have \begin{equation} \lim_{r\to 0} \Big(\underset{x\in D}{\sup }\frac{1}{h(x)} \int_{B(x_0,r)\cap D}G_D^{\alpha }(x,y)h(y)|q(y)|dy\Big)=0. \label{e2.3} \end{equation} \item[(iii)] The function $x\to (\delta (x))^{\alpha -1}q(x)$ is in $L^{1}(D)$. \end{itemize} \end{proposition} The next two Lemmas will play a special role. \begin{lemma}[\cite{CMM}] \label{lem2.2} Let $q$ be a nonnegative function in $K_{\alpha }(D)$ and $h$ be a positive finite $\alpha$-superharmonic function with respect to $X^D$. Then for all $x\in D$, such that $00$. By \eqref{e2.3}, there exists $r>0$ such that \[ \sup_{z\in D}\frac{1}{M_D^{\alpha }\varphi (z)}\int_{B(x_0,2r) \cap D}G_D^{\alpha }(z,y)M_D^{\alpha }\varphi (y)q(y)dy\leq \frac{ \varepsilon }{2}. \] If $x_0\in D$ and $x,x'\in B(x_0,r)\cap D$, then for $| f| \leq q$, we have \begin{align*} &\Big| \int_{D}\frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}M_D^{\alpha }\varphi (y)f(y)dy-\int_{D}\frac{ G_D^{\alpha }(x',y)}{M_D^{\alpha }\varphi (x')} M_D^{\alpha }\varphi (y)f(y)dy\Big| \\ &\leq \int_{D} \big| \frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}-\frac{G_D^{\alpha }(x',y)}{M_D^{\alpha }\varphi (x')}\big| M_D^{\alpha }\varphi (y)q(y)dy \\ &\leq 2\underset{z\in D}{\sup }\int_{B(x_0,2r)\cap D}\frac{1}{ M_D^{\alpha }\varphi (z)}G_D^{\alpha }(z,y)M_D^{\alpha }\varphi (y)q(y)dy \\ &\quad +\int_{(| x_0-y| \geq 2r)\cap D}\big| \frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}-\frac{ G_D^{\alpha }(x',y)}{M_D^{\alpha }\varphi (x')} \big| M_D^{\alpha }\varphi (y)q(y)dy \\ &\leq \varepsilon +\int_{(| x_0-y| \geq 2r)\cap D}\big| \frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}- \frac{G_D^{\alpha }(x',y)}{M_D^{\alpha }\varphi (x')} \big| M_D^{\alpha }\varphi (y)q(y)dy. \end{align*} On the other hand, for every $y\in B^{c}(x_0,2r)\cap D$ and $ x,x'\in B(x_0,r)\cap D$, by using \eqref{e2.5} and the fact that $M_D^{\alpha }\varphi (z)\approx (\delta (z)) ^{\frac{\alpha}{2}-1}$, we have \[ \big| \frac{1}{M_D^{\alpha }\varphi (x)}G_D^{\alpha }(x,y)-\frac{1}{ M_D^{\alpha }\varphi (x')}G_D^{\alpha }(x',y)\big| M_D^{\alpha }\varphi (y)\leq C(\delta (y)) ^{\alpha -1}. \] Now since $x\to \frac{1}{M_D^{\alpha }\varphi (x)} G_D^{\alpha }(x,y)$ is continuous outside the diagonal and $q\in K_{\alpha }(D)$, we deduce by the dominated convergence theorem and Proposition \ref{prop2.1} (iii), that \[ \int_{(| x_0-y| \geq 2r)\cap D}| \frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}-\frac{G_D^{\alpha }(x',y)}{M_D^{\alpha }\varphi (x')}| M_D^{\alpha }\varphi (y)q(y)dy\to 0\quad \text{as } |x-x'| \to 0. \] If $x_0\in \partial D$ and $x\in B(x_0,r)\cap D$, then we have \[ \big| \int_{D}\frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}M_D^{\alpha }\varphi (y)f(y)dy\big| \leq \frac{\varepsilon }{2}+\int_{(| x_0-y| \geq 2r)\cap D}\frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}M_D^{\alpha }\varphi (y)q(y)dy. \] Now, since $\frac{G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}\to 0$ as $| x-x_0| \to 0$, for $| x_0-y| \geq 2r$, then by same argument as above, we obtain \[ \int_{(| x_0-y| \geq 2r)\cap D}\frac{ G_D^{\alpha }(x,y)}{M_D^{\alpha }\varphi (x)}M_D^{\alpha }\varphi (y)q(y)dy\to 0\quad \text{as }| x-x_0| \to 0. \] Consequently, by Ascoli's theorem, we deduce that $\Lambda _{q}$ is relatively compact in $C_0(D)$. \end{proof} \section{Proofs of Theorems \ref{thm1.2} and \ref{thm1.3}} The next Lemma will be used for uniqueness. \begin{lemma}[{\cite[Lemma 4]{CMM}}] \label{lem3.1} Let $h\in B^{+}(D)$ and $\upsilon $ be a nonnegative $\alpha$-superhar\-monic function on $D$ with respect to $X^{D}$. Let $z$ be a Borel measurable function in $D$ such that $V(h| z| )<\infty $ and $\upsilon =z+V(hz)$. Then $z$ satisfies \[ 0\leq z\leq \upsilon . \] \end{lemma} \begin{proof}[Proof of Theorem \ref{thm1.2}] Let $\varphi $ be a positive continuous function on $\partial D$. We recall that on $D$ we have \[ M_D^{\alpha }\varphi (x)\approx M_D^{\alpha }1(x)\approx (\delta (x))^{\frac{\alpha}{2} -1}. \] Let $\widetilde{p_0}=\gamma (M_D^{\alpha }\varphi )^{\gamma -1}p_0$ and put $c_0=e^{-a_{\alpha }(\widetilde{p_0} )}$, where $a_{\alpha }(\widetilde{p_0})$ is given by Proposition \ref{prop2.1}(i). Since by (H1), $\widetilde{p_0}\in K_{\alpha}(D)$, it follows from Proposition \ref{prop2.1} that $V(\widetilde{p_0})\leq a_{\alpha }(\widetilde{p_0})<\infty $. Define the nonemty closed bounded convex $\Lambda $ by \[ \Lambda =\{\omega \in B^{+}(D):\text{ }c_0\leq \omega \leq 1\}. \] Let $T$ be the operator defined on $\Lambda $ by \[ T\omega :=1-\frac{1}{M_D^{\alpha }\varphi }V_{\widetilde{p_0}}( \widetilde{p_0}M_D^{\alpha }\varphi )+\frac{1}{M_D^{\alpha }\varphi } V_{\widetilde{p_0}}(\widetilde{p_0}\omega M_D^{\alpha}\varphi -p_0(\omega M_D^{\alpha}\varphi )^{\gamma }). \] We claim that $T$ maps $\Lambda $ to itself. Indeed, for each $ \omega \in \Lambda $ we have \[ T\omega \leq 1-\frac{1}{M_D^{\alpha}\varphi }V_{\widetilde{p_0} }(p_0(\omega M_D^{\alpha}\varphi )^{\gamma })\leq 1. \] On the other hand, since the function $\widetilde{p_0}\omega M_D^{\alpha}\varphi -p_0(\omega M_D^{\alpha}\varphi )^{\gamma }\geq 0$, we deduce by Lemma \ref{lem2.2} with $h=M_D^{\alpha}\varphi $, that $ T\omega \geq 1-\frac{1}{M_D^{\alpha}\varphi }V_{\widetilde{p_0}}( \widetilde{p_0}M_D^{\alpha}\varphi )\geq c_0$. Hence $T\Lambda \subset \Lambda $. Next, we aim at proving that $T$ is nondecreasing on $ \Lambda $. To this end, we let $\omega _1$, $\omega _2\in \Lambda $ such that $\omega _1\leq \omega _2$. Using the fact that the function $ t\to \gamma t-t^{\gamma }$ is nondecreasing on $[0,1]$, we deduce that \begin{align*} & T\omega _2-T\omega _1 \\ &=\frac{1}{M_D^{\alpha}\varphi }V_{\widetilde{ p_0}}(\widetilde{p_0}\omega _2 M_D^{\alpha}\varphi -p_0(\omega _2M_D^{\alpha}\varphi )^{\gamma })-\frac{1}{M_D^{\alpha}\varphi }V_{ \widetilde{p_0}}(\widetilde{p_0}\omega _1M_D^{\alpha}\varphi -p_0(\omega _1M_D^{\alpha}\varphi )^{\gamma }) \\ &=\frac{1}{M_D^{\alpha}\varphi }V_{\widetilde{p_0}}(p_0( M_D^{\alpha}\varphi )^{\gamma }[ (\gamma \omega _2-\omega _2^{\gamma })-(\gamma \omega _1-\omega _1^{\gamma })] ) \geq 0. \end{align*} Next we define the sequence $(\omega _{k})_{k\geq 0}$ by \begin{gather*} \omega _0=1-\frac{1}{M_D^{\alpha}\varphi }V_{\widetilde{p_0}}( \widetilde{p_0}M_D^{\alpha}\varphi ), \\ \omega _{k+1}=T\omega _{k}. \end{gather*} Clearly $\omega _0\in \Lambda $ and $\omega _1=T\omega _0\geq \omega _0$. Thus, from the monotonicity of $T$, we deduce that \[ c_0\leq \omega _0\leq \omega _1\leq ...\leq \omega _{k}\leq 1. \] So, the sequence $(\omega _{k})_{k\geq 0}$ converges to a measurable function $\omega \in \Lambda $. Therefore by applying the monotone convergence theorem, we obtain \[ \omega =1-\frac{1}{M_D^{\alpha}\varphi }V_{\widetilde{p_0}}(\widetilde{ p_0}M_D^{\alpha}\varphi )+\frac{1}{M_D^{\alpha}\varphi }V_{ \widetilde{p_0}}(\widetilde{p_0}\omega M_D^{\alpha}\varphi -p_0(\omega M_D^{\alpha}\varphi )^{\gamma }) \] Put $u=\omega M_D^{\alpha}\varphi $. Then we have \begin{equation} u=M_D^{\alpha}\varphi -V_{\widetilde{p_0}}(\widetilde{p_0}M_D^{\alpha }\varphi )+V_{\widetilde{p_0}}(\widetilde{p_0}u-p_0u^{\gamma }) \label{e3.1} \end{equation} or equivalently \begin{equation} u-V_{\widetilde{p_0}}(\widetilde{p_0}u)=M_D^{\alpha}\varphi -V_{ \widetilde{p_0}}(\widetilde{p_0}M_D^{\alpha}\varphi )-V_{\widetilde{ p_0}}(p_0u^{\gamma }). \label{e3.2} \end{equation} Observe that by Proposition \ref{prop2.1} (ii), we have $V(\widetilde{p_0}u)<\infty $. So applying the operator $(I+V(\widetilde{p_0}.))$ on both sides of \eqref{e3.2}, we deduce by using \eqref{e1.8} and \eqref{e1.9} that \[ u=M_D^{\alpha}\varphi -V(p_0u^{\gamma }). \] Now using (H1) and similar argument as in the proof of Lemma \ref{lem2.3}, we prove that $x\mapsto \frac{1}{M_D^{\alpha }\varphi }V(p_0u^{\gamma })\in C_0(D)$. So $u$ is a continuous function in $D$ and $u$ is a solution of \eqref{e1.3}. It remains to prove the uniqueness of such a solution. Let $u$ be a continuous solution of \eqref{e1.3}. Since the function $x\mapsto \frac{u(x)}{M_D^{\alpha}1(x)}$ is continuous and positive in $D$ such that $\underset{x\to z\in \partial D}{\lim }\frac{u(x)}{M_D^{\alpha}1(x)}=\varphi (z)$, it follows that $u(x)\approx M_D^{\alpha}1(x)\approx M_D^{\alpha}\varphi (x)$. Then by using this fact and Lemma \ref{lem2.3}, we have \begin{gather*} (-\Delta )^{\alpha/2}(u+V(p_0u^{\gamma }))=0 \quad \text{in }D, \\ \lim_{x\to z\in \partial D} \frac{(u+V(p_0u^{\gamma}))(x)}{M_D^{\alpha}1(x)} =\varphi (z) . \end{gather*} So from the uniqueness of problem \eqref{e1.5} (see \cite{CMM}]), we deduce that \[ u+V(p_0u^{\gamma })=M_D^{\alpha}\varphi \text{ in }D. \] It follows that if $u$ and $v$ are two continuous solution of $( \ref{e1.3})$, then $z=v-u$ satisfies \[ z+V(p_0hz)=0\quad \text{in }D, \] where $h$ is the nonnegative measurable function defined in $D$ by \[ h(x)=\begin{cases} \frac{v^{\gamma }-u^{\gamma }}{v-u}, & \text{if }u(x)\neq v(x), \\ 0, & \text{if }u(x)=v(x). \end{cases} \] Since $V(p_0h| z| )<\infty $, we deduce by Lemma \ref{lem3.1} that $z=0$, and so $u=v$. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.3}] Let $\widetilde{p}=\sigma (M_D^{\alpha}\varphi ) ^{\sigma -1}(M_D^{\alpha}\psi )^{r}p$, $\widetilde{q} =\beta (M_D^{\alpha}\psi )^{\beta -1}(M_D^{\alpha }\varphi )^{s}q$. Then by hypothesis $(\mathbf{H}_2)$, $\widetilde{p}$ and $\widetilde{q}$ are $ K_{\alpha}(D)$. Put $c_1=e^{-a_{\alpha }(\widetilde{p})}$, $c_2=e^{-a_{\alpha }(\widetilde{q})}$. Note that by Proposition \ref {prop2.1}, we have $a_{\alpha }(\widetilde{p})<\infty $ and $a_{\alpha }( \widetilde{q})<\infty $. Consider the nonemty closed convex set $\Gamma $ defined by \[ \Gamma =\{(y,z)\in C(\overline{D})\times C(\overline{D}):\text{ \ }c_1\leq y\leq 1\text{ \ and }c_2\leq z\leq 1\}. \] Let $T$ be the operator defined on the set $\Gamma $ by $T(y,z):=(\omega ,\theta )$, such that \linebreak $(\widetilde{u}=\omega M_D^{\alpha}\varphi ,\widetilde{v} =\theta M_D^{\alpha}\psi )$ is the unique positive continuous solution of the problem \begin{gather*} (-\Delta )^{\alpha/2}\widetilde{u}+(( M_D^{\alpha}\psi )^{r}z^{r}p)(x)\widetilde{u}^{\sigma }=0 \quad \text{ in }D, \\ (-\Delta )^{\alpha/2}\widetilde{v}+(( M_D^{\alpha}\varphi )^{s}y^{s}q)(x)\widetilde{v}^{\beta }=0 \quad \text{in }D, \\ \lim_{x\to z\in \partial D} \frac{\widetilde{u}(x)}{ M_D^{\alpha}1(x)}=\varphi (z), \\ \lim_{x\to z\in \partial D} \frac{\widetilde{v}(x)}{ M_D^{\alpha}1(x)}=\psi (z), \end{gather*} According to Theorem \ref{thm1.2}, we have \begin{gather*} \omega =1-\frac{1}{M_D^{\alpha}\varphi }V(z^{r}\omega ^{\sigma }( M_D^{\alpha}\psi )^{r}(M_D^{\alpha}\varphi ) ^{\sigma }p), \\ \theta =1-\frac{1}{M_D^{\alpha}\psi }V(y^{s}\omega ^{\beta }( M_D^{\alpha}\varphi )^{s}(M_D^{\alpha}\psi ) ^{\beta }q). \end{gather*} Moreover we have $c_1\leq \omega \leq 1$ and $c_2\leq \theta \leq 1$ and by Lemma \ref{lem2.3}, $T(\Gamma )$ is equicontinuous on $\overline{D}$. Since $T(\Gamma )$ is also bounded, then we deduce that $ T(\Gamma )$ is relatively compact in $C(\overline{D})\times C(\overline{D})$. This implies in particular that $T(\Gamma )\subset \Gamma $. Next, we shall prove the continuity of the operator $T$ in $\Gamma $ in the supremum norm. Let $(y_{k},z_{k})_{k}$ be a sequence in $\Gamma $ which converges uniformly to a function $(y,z)$ in $\Gamma $. Put $(\omega _{k},\theta _{k})=T(y_{k},z_{k})$ and $(\omega ,\theta )=T(y,z)$. Then we have \begin{align*} | \omega _{k}-\omega | &=\Big| \frac{1}{M_D^{\alpha }\varphi }V(z^{r}\omega ^{\sigma }(M_D^{\alpha}\psi ) ^{r}(M_D^{\alpha}\varphi )^{\sigma }p)-\frac{1}{M_D^{\alpha }\varphi }V(z_{k}^{r}\omega _{k}^{\sigma }(M_D^{\alpha}\psi )^{r}(M_D^{\alpha}\varphi )^{\sigma }p)\Big| \\ &\leq \frac{1}{\sigma M_D^{\alpha}\varphi }V(| z^{r}\omega ^{\sigma }-z_{k}^{r}\omega _{k}^{\sigma }| (M_D^{\alpha}\varphi )\widetilde{p}). \end{align*} Using the fact that $| z^{r}\omega ^{\sigma }-z_{k}^{r}\omega _{k}^{\sigma }| \leq 2$ and that $\widetilde{p} \in K_{\alpha }(D)$, we deduce by Proposition \ref{prop2.1} and the dominated convergence theorem, that $\omega _{k}\to \omega $ as $k\to \infty $. Similarly we prove that $\theta _{k}\to \theta $ as $k\to \infty $. So $T(y_{k},z_{k})\to T(y,z)$ as $k\to \infty $. Since $T(\Gamma )$ is relatively compact in $C(\overline{D})\times C(\overline{D})$, we deduce that \[ \| T(y_{k},z_{k})-T(y,z)\| _{\infty }\to 0\quad \text{as }k\to \infty . \] Now the Schauder fixed point theorem implies that there exists $(y,z)\in \Gamma $ such that $T(y,z)=(y,z)$. Which is equivalent to \begin{gather*} u=M_D^{\alpha}\varphi -V(pu^{\sigma }v^{r}), \\ v=M_D^{\alpha}\psi -V(qu^{s}v^{\beta }), \end{gather*} where $(u,v)=(yM_D^{\alpha}\varphi ,zM_D^{\alpha}\psi)$. The pair $(u,v)$ is a required solution of \eqref{S1} in the sense of distributions. This completes the proof. \end{proof} \begin{thebibliography}{99} \bibitem{Als} R. S. Alsaedi; \emph{Positive Solutions for Some Nonlinear Elliptic Systems in Exterior Domains of ${\mathbb{R}}^2$}, Abstract and Applied Analysis, Volume 2012, Article ID 273017, 13 pages. \bibitem{AMZ} R.S. Alsaedi, H. M\^{a}agli, N. Zeddini: \emph{Positive Solutions for Some Competetive Elliptic Systems}, to appear in Mathematica Slovaca. \bibitem{Ap} D. Applebaum; \emph{L\'evy processes-from probability to finance and quantum groups}, Notices Amer. Math. Soc., 51 (2004), 1336-1347. \bibitem{B} K. Bogdan; \emph{Representation of $\alpha$-harmonic functions in Lipschitz domains}, Hiroshima Math. J. 29 (1999), 227-243. \bibitem{BB} K. Bogdan, T. Byczkowski; \emph{Potential theory for the $\alpha$-stable Schr\"{o}dinger operator on bounded Lipschitz domains}, Studia Math. 133 (1999), 53-92. \bibitem{BH}J. Bliedtner, W. Hansen; \emph{Potential Theory. An Analytic and Probabilistic Approach to Balayage,} Springer Berlin, 1986. \bibitem{CMM} R. Chemmam, H. M\^{a}agli, S. Masmoudi; \emph{On a new Kato class and positive solutions of Dirichlet problems for the fractional Laplacian in bounded domains,} Nonlinear Anal. 74 (2011), 1555-1576. \bibitem{CS1} Z. Q. Chen, R. Song; \emph{Estimates on Green functions and Poisson kernels for symmetric stable processes}, Math. Ann. 312 (1998) 465-501. \bibitem{CS2} Z. Q. Chen, R. Song; \emph{Martin boundary and integral representation for harmonic function of symmetric processes}, J. Funct. Anal. 159 (1998) 267-294. \bibitem{CS3} Z. Q. Chen, R. Song; \emph{General gauge and conditional gauge theorems}, Ann. Probab. 30 (2002) 1313-1339. \bibitem{CK} Z. Q. Chen, P. Kim; \emph{Stability of Martin boundary under nonlocal Feynman–Kac perturbations}, Probab. Theory Related Fields 128 (2004) 525-564. \bibitem{CZ} K.L. Chung, Z. Zhao; \emph{From Brownian Motion to Schrodinger's Equation}, Springer Verlag (1995). \bibitem{CR} F .C. Cirstea, V. D. Radulescu; \emph{Entire solutions blowing up at infinity for semilinear elliptic systems}, J. Math. Pures Appl. \textbf{81} (2002), 827--846. \bibitem{DPV} E. Di Nezza, G. Palatucci, E. Valdinoci; \emph{Hitchhiker’s guide to the fractional Sobolev spaces}, Bull. Sci. Math. 136 (2012), 521-573. \bibitem{GMRZ} A. Ghanmi, H. M\^{a}agli, V. Radulescu, N. Zeddini; \emph{Large and bounded solutions for a class of nonlinear Schr\"{o}dinger stationary Systems}, Analysis and Applications \textbf{7} (2009), 391-401. \bibitem{GMTZ}A. Ghanmi, H. M\^{a}agli, S. Turki, N. Zeddini; \emph{Existence of Positive Bounded Solutions for Some Nonlinear Elliptic Systems}, J. Math. Anal. Appl. \textbf{352} (2009), 440--448. \bibitem{GR} M. Ghergu, V. D. Radulescu; \emph{On a class of singular Gierer--Meinhart systems arising in morphogenesis}, C. R. Math. Acad. Sci. Paris. Ser. I \textbf{344} (2007), 163--168. \bibitem{JW} A. Janicki, A. Weron: \emph{Simulation and Chaotic Behavior of $\alpha$-Stable Processes}, Dekker, 1994 \bibitem{KSZ} J. Klater, M. F. Shlesinger, G. Zumofen; \emph{Beyond Brownian motion}, Phys. Today 49 (2) (1996) 33–39. \bibitem{LW1} A. V. Lair, A. W. Wood; \emph{Large solutions of semilinear elliptic equations with nonlinear gradient terms}, Int. J. Math. Sci. \textbf{22} (1999), 869-883. \bibitem{LW2} A. V. Lair, A. W. Wood; \emph{Existence of entire large positive solutions of semilinear elliptic systems}, J. Differential Equations (2) \textbf{164} (2000), 380--394. \bibitem{M} H. M\^{a}agli; \emph{Perturbation Semi-lin\'{e}aire des R\'{e}solvantes et des Semi-groupes}, Potential Analysis. vol. \textbf{3} (1994), 61-87. \bibitem{MZ} H. M\^{a}agli, M. Zribi; \emph{On a new Kato class and singular solutions of a nonlinear elliptic equation in bounded domains of }$\mathbb{R}^{n}$, Positivity. \textbf{9\ }(2005), 667-686. \bibitem{MHTL} C. Mu, S. Huang, Q. Tian, L. Liu; \emph{Large solutions for an elliptic system of competetive type: Existence, uniqueness and asymptotic behavior}, Nonlinear Anal. 71 (2009) 4544-4552. \bibitem{P} A. Pazy; \emph{Semigroups of Linear Operators and Applications to Partial Differential Equations}, Springer, New York, 1983. \bibitem{W} D. V. Widder; \emph{The Laplace Transform}, Princeton Univ. Press, Princeton, NJ, 1941. \bibitem{Y} K. Yosida; \emph{Functional Analysis}, 6th Edition, Springer, Berlin, 1980. \end{thebibliography} \end{document}