\documentclass[reqno]{amsart} \usepackage{hyperref} \AtBeginDocument{{\noindent\small \emph{Electronic Journal of Differential Equations}, Vol. 2011 (2011), No. 81, pp. 1--13.\newline ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu \newline ftp ejde.math.txstate.edu} \thanks{\copyright 2011 Texas State University - San Marcos.} \vspace{9mm}} \begin{document} \title[\hfilneg EJDE-2011/81\hfil Existence of continuous positive solutions] {Existence of continuous positive solutions for some nonlinear polyharmonic systems outside the unit ball} \author[S. Turki\hfil EJDE-2011/81\hfilneg] {Sameh Turki} \address{Sameh Turki \newline D\'epartement de Math\'ematiques, Facult\'e des Sciences de Tunis, Campus Universitaire, 2092 Tunis, Tunisia} \email{sameh.turki@ipein.rnu.tn} \thanks{Submitted May 23, 2011. Published June 21, 2011.} \subjclass[2000]{34B27, 35J40} \keywords{Polyharmonic elliptic system; Positive solutions; Green function; \hfill\break\indent polyharmonic Kato class} \begin{abstract} We study the existence of continuous positive solutions of the m-polyharmonic nonlinear elliptic system \begin{gather*} (-\Delta)^{m}u+\lambda p(x)g(v)=0,\\ (-\Delta )^{m}v+\mu q(x)f(u)=0 \end{gather*} in the complement of the unit closed ball in $\mathbb{R}^{n}$ $(n>2m$ and $m\geq 1$). Here the constants $\lambda,\mu$ are nonnegative, the functions $f,g$ are nonnegative, continuous and monotone. We prove two existence results for the above system subject to some boundary conditions, where the nonnegative functions $p,q$ satisfy some appropriate conditions related to a Kato class of functions. \end{abstract} \maketitle \numberwithin{equation}{section} \newtheorem{theorem}{Theorem}[section] \newtheorem{proposition}[theorem]{Proposition} \newtheorem{example}[theorem]{Example} \newtheorem{definition}[theorem]{Definition} \allowdisplaybreaks \section{Introduction} In this article, we discuss the existence of positive continuous solutions (in the sense of distributions) for the $m$-polyharmonic nonlinear elliptic system \begin{equation} \begin{gathered} (-\Delta)^{m} u+\lambda p(x)g(v)=0,\quad x\in D, \\ (-\Delta)^{m} v+\mu q(x)f(u)=0,\quad x\in D, \\ \lim_{x\to\xi \in\partial D} \frac{u(x)}{(|x|^2-1)^{m-1}}=a\varphi(\xi) ,\quad \lim_{x\to\xi \in\partial D}\frac{v(x)}{(|x|^2-1)^{m-1}}=b\psi(\xi), \\ \lim_{|x|\to \infty}\frac{u(x)}{(|x|^2-1)^{m-1} }=\alpha ,\quad \lim_{|x|\to \infty}\frac{v(x)}{(|x|^2-1)^{m-1} }=\beta , \end{gathered} \label{P} \end{equation} where $D$ is the complementary of the unit ball in $\mathbb{R}^n$ ($n> 2m $) and m is a positive integer. The constants $\lambda,\mu$ are nonnegative, $f, g:(0,\infty )\to [0,\infty )$ are monotone and continuous and $p,q:D\to[0,\infty)$ are measurable functions. Also we fix two nontrivial nonnegative continuous functions $\varphi$ and $\psi$ on $\partial D$ and the constants $a,b,\alpha,\beta$ are nonnegative and satisfy $a+\alpha>0$, $b+\beta>0$. Since our tools are based on potential theory approach, we denote by $G_{m,n}^B$ the Green function of $(-\Delta )^{m}$ on the unit ball $B$ in $\mathbb{R}^n$ $(n\geq 2)$ with Dirichlet boundary conditions $(\frac{\partial}{\partial\nu})^j u=0$, $0\leq j \leq m-1$ and where $\frac{\partial}{\partial\nu}$ is the outward normal derivative. Boggio \cite{Boggio} obtained an explicit expression for $G_{m,n}^B$ given by \begin{equation} G_{m,n}^B(x,y)=k_{m,n}| x-y|^{2m-n}\int_{1}^{\frac{[ x,y] }{| x-y| }}\frac{(r ^{2}-1)^{m-1}}{r ^{n-1} }dr , \label{1.2} \end{equation} where $k_{m,n}$ is a positive constant and $[ x,y]^{2}=|x-y|^{2}+(1-|x|^{2})(1-|y|^{2})$, for $x,y\in B$. It is obvious that the positivity of $G_{m,n}^B$ holds in $B$ but this does not hold in an arbitrary bounded domain (see for example \cite{Garabedian}). For $m=1$, we do not have this restriction. Putting $$ [ x,y] ^{2}=|x-y|^{2}+(|x|^{2}-1)(|y|^{2}-1), $$ for $x,y\in D$ and denote by $G_{m,n}^D$ the Green function of $(-\Delta )^{m}$ in $D$ with Dirichlet boundary conditions $(\frac{\partial}{\partial\nu} )^ju=0$, $0\leq j \leq m-1$, then $G_{m,n}^D$ has the same expression defined by \eqref{1.2}. That is, \[ G_{m,n}^D(x,y)=k_{m,n}| x-y|^{2m-n} \int_{1}^{\frac{[x,y] }{| x-y| }}\frac{(r ^{2}-1)^{m-1}}{r ^{n-1} }d r ,\quad \text{for }x,y \in D. \] In \cite{BMZ}, the authors proved some estimates for $G_{m,n}^D$. In particular, they showed that there exists $C_0>0$ such that for each $x,y,z \in D$, we have \[ \frac{G_{m,n}^D(x,z)G_{m,n}^D(z,y)}{G_{m,n}^D(x,y)} \leq C_0\big[ (\frac{ \rho (z)}{\rho (x)})^{m}G_{m,n}^D(x,z)+(\frac{\rho (z)}{ \rho (y)})^{m}G_{m,n}^D(y,z)\big], \] where throughout this paper, $\rho(x)=1-\frac{1}{|x|}$, for all $x\in D$. This form is called the $3G$-inequality and has been exploited to introduce the polyharmonic Kato class $K_{m,n}^{\infty}(D)$ which is defined as follows \begin{definition}[\cite{BMZ}] \label{def1.1} \rm A Borel measurable function $q$ in $D$ belongs to the Kato class $K_{m,n}^{\infty}(D)$ if $q$ satisfies \begin{gather*} \lim_{\alpha \to 0}\Big(\sup _{x\in D} \int_{D\cap B(x,\alpha )}(\frac{\rho (y)}{\rho (x)}) ^{m}G_{m,n}^D(x,y)| q(y)| dy\Big)=0, \\ \lim_{M \to \infty}\Big(\sup_{x\in D} \int_{(|y|\geq M)}(\frac{\rho (y)}{\rho (x)}) ^{m}G_{m,n}^D(x,y)| q(y)| dy\Big)=0. \end{gather*} \end{definition} This class is well studied when $m=1$ in \cite{BMZm1}. As a typical example of functions belonging to the class $K_{m,n}^{\infty}(D)$, we quote an example from \cite{BMZ}: Let $\gamma,\nu\in\mathbb{R}$ and $q$ be the function defined in $D$ by $q(x)=\frac{1}{|y|^{\nu-\gamma}(|y|-1)^\gamma}$. Then $$ q\in K_{m,n}^{\infty}(D) \Leftrightarrow \gamma<2m<\nu . $$ Our main purpose in this paper is to study problem \eqref{P} when $p$ and $q$ satisfy an appropriate condition related to the Kato class $K_{m,n}^{\infty}(D)$ and to investigate the existence and the asymptotic behavior of such positive solutions. For this aim we shall refer to the bounded continuous solution $H_{D}\varphi $ of the Dirichlet problem (see \cite{Armitage}) \begin{gather*} \Delta u=0\quad \text{in } D, \\ \lim_{x\to\xi \in\partial D}u(x)=\varphi(\xi),\quad \lim_{|x|\to\infty }u(x) =0, \end{gather*} where $\varphi$ is a nonnegative nontrivial continuous function on $\partial D$. Also, we refer to the potential of a measurable nonnegative function $f$, defined in $D$ by \[ V_{m,n}f(x)=\int_{D}G_{m,n}^D(x,y)f(y)dy. \] The outline of our article is as follows. In Section 2, we recapitulate some properties of functions belonging to $K_{m,n}^{\infty}(D)$ developed in \cite{BMZ} and adopted to our interest. In Section 3, we aim at proving a first existence result for \eqref{P}. In fact, let $a,b,\alpha,\beta$ be nonnegative real numbers with $a+\alpha>0$, $b+\beta>0$ and $\varphi,\psi$ are nontrivial nonnegative continuous functions on $\partial D$. Let $h$ be the harmonic function defined in $D$ by $h(x)= 1-\frac{1}{|x|^{n-2}}$. Let $\theta$ and $\omega$ be the functions defined in $D$ by \begin{gather*} \theta(x)=\gamma(x)(\alpha\,h(x)\,+\,a\,H_D\varphi(x) ),\\ \omega(x)=\gamma(x)(\beta\,h(x)\,+\,b\,H_D\psi(x) ), \end{gather*} where $\gamma(x)=(|x|^2-1)^{m-1}$. The functions $f,g,p$ and $q$ are required to satisfy the following hypotheses. \begin{itemize} \item[(H1)] $f$, $g:(0,\infty )\to [0,\infty ) $ are nondecreasing and continuous; \item[(H2)] \begin{gather*} \lambda_0:=\inf_{x\in D} \frac{\theta(x)}{V_{m,n}(p g (\omega))(x)}> 0,\\ \mu_0:=\inf_{x\in D} \frac{\omega(x)}{V_{m,n} (q f (\theta))(x)}> 0; \end{gather*} \item[(H3)] The functions $p$ and $q$ are measurable nonnegative and satisfy \[ x\to\tilde{p}(x)=\frac{p(x)\,g(\omega(x))}{\gamma(x)} \quad\text{and}\quad x\to\tilde{q}(x)=\frac{q(x)\,f(\theta(x))}{\gamma(x)} \] belong to the Kato class $K_{m,n}^{\infty }(D)$. \end{itemize} Then we prove the following result. \begin{theorem} \label{thm1.2} Assume {\rm (H1)--(H3)}. Then for each $\lambda \in [ 0,\lambda _0)$ and each $ \mu \in [ 0,\mu _0)$, problem \eqref{P} has a positive continuous solution $(u,v)$ that for each $x\in D $ satisfies \begin{gather*} (1-\frac{\lambda }{\lambda_0})\theta(x)\leq u(x) \leq \theta(x), \\ (1-\frac{\mu}{\mu _0})\omega(x)\leq v(x) \leq \omega(x) . \end{gather*} % \label{21} \end{theorem} Next, we establish a second existence result for problem \eqref{P} where $a=b=\lambda=\mu=1$. Namely, we study the system \begin{equation} \begin{gathered} (-\Delta)^{m} u+ p(x)g(v)=0,\quad x\in D \quad\text{(in the sense of distributions}), \\ (-\Delta)^{m} v+ q(x)f(u)=0,\quad x\in\,D,\\ \lim_{x\to\xi \in\partial D}\frac{u(x)}{(|x|^2-1)^{m-1}}=\varphi(\xi) , \quad \lim_{x\to\xi \in\partial D}\frac{v(x)}{(|x|^2-1)^{m-1}}=\psi(\xi) , \\ \lim_{|x|\to \infty}\frac{u(x)}{(|x|^2-1)^{m-1} }=\alpha ,\quad \lim_{|x|\to \infty}\frac{v(x)}{(|x|^2-1)^{m-1} }=\beta . \end{gathered} \label{Q} \end{equation} To study this problem, we fix a positive continuous function $\phi$ on $\partial D$. We put $\rho_0= \gamma h_0 $, where $h_0=H_D\phi $ and we assume the following hypotheses: \begin{itemize} \item[(H4)] The functions $f,g:(0,\infty)\to[0,\infty)$ are nonincreasing and continuous; \item[(H5)] The functions $p_1:=p\frac{g(\rho_0)}{\rho_0}$ and $q_1:=q \frac{f(\rho_0)}{\rho_0}$ belong to the Kato class $K_{m,n}^{\infty }(D)$. \end{itemize} Here, we mention that the method used to prove Theorem \ref{thm1.3} stated below is different from that in Theorem \ref{thm1.2}. In fact, with loss of $\lambda$ and $\mu$, the boundary $\partial D$ will play a capital role to construct a positive and continuous solution for \eqref{Q} by means of a fixed point argument. Our second existence result is the following. \begin{theorem} \label{thm1.3} Assume {\rm (H4)--(H5)}. Then there exists a constant $c>1 $ such that if $\varphi \geq c \phi$ and $\psi \geq c \phi$ on $\partial D$, then problem \eqref{Q} has a positive continuous solution $(u,v)$ that for each $x\in D$ satisfies \begin{gather*} (|x|^2-1)^{m-1}(\alpha\,h(x)+h_0(x))\leq u(x) \leq (|x|^2-1)^{m-1}(\alpha\,h(x)+H_D\varphi(x)),\\ (|x|^2-1)^{m-1}(\beta\,h(x)+h_0(x))\leq v(x) \leq (|x|^2-1)^{m-1}(\beta\,h(x)+H_D\psi(x)). \end{gather*} \end{theorem} This result is a follow up to the one obtained by Athreya \cite{Athr}. For $m=1$, the existence of solutions for nonlinear elliptic systems has been extensively studied for both bounded and unbounded $C^{1,1}$-domains in $\mathbb{R}^{n}$ ($n\geq 3$) (see for example \cite{cirstea,david89,ghanmi,GMTZ,ghergu1,ghergu2,ghergu3,lairwood,zhou}). The motivation for our study comes from the results proved in \cite{GMTZ} and which correspond to the case $m=1$ in this article. Section 4 gives some examples where hypotheses (H2) and (H3) are satisfied and to illustrate Theorem \ref{thm1.3}. In the sequel and in order to simplify our statements we denote by $C$ a generic positive constant which may vary from line to line and for two nonnegative functions $f$ and $g$ on a set $S$, we write $f(x)\asymp g(x)$, for $x \in S$, if there exists a constant $C>0$ such that $g(x)/C\leq f(x)\leq Cg(x)$ for all $x \in S$. Let $$ C_0(D):=\{f\in C(D): \lim_{|x|\to 1}f(x)= \lim_{|x|\to \infty}f(x)=0 \}. $$ \section{Preliminary results} In this section, we are concerned with some results related to the Kato class $K_{m,n}^{\infty }(D)$ which are useful for the proof of our main results stated in Theorems \ref{thm1.2} and \ref{thm1.3}. \begin{proposition}[\cite{BMZ}] \label{prop2.1} Let $q$ be a function in $ K_{m,n}^{\infty }(D)$, then $$ \|q\|_{D}:= \sup_{x\in D} \int_{D}(\frac{\rho (y)}{\rho (x)}) ^{m}G_{m,n}^D(x,y)| q(y)| dy<\infty. $$ \end{proposition} To present the following Proposition, we need to denote by $ \mathcal{H} $ the set of nonnegative harmonic functions $h$ defined in $D $ by $$ h(x)= \int_{\partial D}P(x,\xi)\nu(d\xi), $$ where $\nu$ is a nonnegative measure on $\partial D$ and $P(x,\xi)=\frac{|x|^2-1}{|x-\xi|^n}$ is the Poisson kernel in $D$. From the 3G-inequality, we derive the following result. \begin{proposition} \label{prop2.2} Let $q$ be a nonnegative function in $ K_{m,n}^{\infty}(D)$. Then we have \begin{itemize} \item[(i)] $$ \alpha_q:=\sup_{x,y \in D}\int_D\frac{G_{m,n}^D(x,z) G_{m,n}^D(z,y)}{G_{m,n}^D(x,y)}q(z)dz<\infty; $$ \item[(ii)] For any function $h\in \mathcal{H}$ and each $x\in D$, we have \[ \int_{D}G_{m,n}^D(x,z)(|z| ^{2}-1)^{m-1}h(z)q(z)dz\leq \alpha_q (|x| ^{2}-1)^{m-1}h(x). \] \end{itemize} \end{proposition} \begin{proof} From the 3G-inequality, there exists $C_0>0$ such that for each $x,y,z\in D$, we have \[ \frac{G_{m,n}^D(x,z)G_{m,n}^D(z,y)}{G_{m,n}^D(x,y)} \leq C_0\big[ (\frac{ \rho (z)}{\rho (x)})^{m}G_{m,n}^D(x,z)+(\frac{\rho (z)}{ \rho (y)})^{m}G_{m,n}^D(y,z)\big] . \] This implies that $\alpha_{q} \leq 2C_0\,{\|q\|}_{D}$. Then the assertion (i) holds from Proposition \ref{prop2.1}. Now, we shall prove (ii). Let $h\in\mathcal{ H}$, then there exists a nonnegative measure $\nu$ on $\partial D$ such that \begin{equation} h(x)= \int_{\partial D}P(x,\xi)\nu(d\xi). \label{M1} \end{equation} On the other hand, by using the transformation $r^2=1+\frac{\varrho(x,y)}{|x-y|^2}(1-t)$ in \eqref{1.2}, where $\varrho(x,y)=[x,y]^2-|x-y|^2=(|x|^2-1)(|y|^2-1)$, we obtain $$ G_{m,n}^D(x,y)=\frac{k_{m,n}}{2}\frac{(\varrho(x,y))^m}{[x,y]^n} \int_0^{1}\frac{(1-t)^{m-1}}{(1-t\frac{\varrho(x,y)}{[x,y]^2}) ^{n/2}}dt. $$ This implies for each $x,z\in D$ and $\xi\in\partial D$ that $$ \lim_{y\to\xi}\frac{G_{m,n}^D(z,y)}{G_{m,n}^D(x,y)} =\frac{(|z|^2-1)^{m-1}P(z,\xi)}{(|x|^2-1)^{m-1}P(x,\xi)}. $$ So, it follows from Fatou's lemma that \begin{align*} &\int_D G_{m,n}^D(x,z) \frac{(|z|^2-1)^{m-1}P(z,\xi)}{(|x|^2-1)^{m-1}P(x,\xi)}q(z)dz\\ &\leq \liminf_{y\to\xi} \int_D\frac{G_{m,n}^D(x,z)G_{m,n}^D(z,y)}{G_{m,n}^D(x,y)}q(z)dz \leq \alpha_{q}. \end{align*} This, together with \eqref{M1}, completes the proof. \end{proof} \begin{proposition}[\cite{BMZ}] \label{prop2.3} Let $q\in K_{m,n}^{\infty }(D)$. Then the function $ z\to \frac{(|z|-1)^{2m-1}}{|z|^{n-1}}q(z)$ is in $L^1(D)$. \end{proposition} \begin{proposition}\cite{BMZ} \label{prop2.4} Let $q\in K_{m,n}^{\infty }(D)$ and $h$ be a bounded function in $\mathcal{H}$. Then the function $$ x\to \int_D \big(\frac{|y|^2-1}{|x|^2-1}\big)^{m-1}G_{m,n}^D (x,y)\,h(y)\,|q(y)|dy $$ lies in $C_0(D)$. \end{proposition} For a nonnegative function $q\in K_{m,n}^{\infty}(D)$, we denote $$ {\mathcal{F}}_q=\{ p \in K_{m,n}^{\infty}(D):|p|\leq q \text{ in }D \}. $$ \begin{proposition}[\cite{BMZ}] \label{prop2.5} For any nonnegative function $q\in K_{m,n}^{\infty}(D)$, the family of functions $$ \big\{\int_D\big(\frac{|y|^2-1}{|x|^2-1} \big)^{m-1}G_{m,n}^D(x,y)h_0(y)p(y)dy,\,p\in {\mathcal{F}}_q \big\} $$ is uniformly bounded and equicontinuous in $\overline{D}\cup \{\infty\}$. Consequently it is relatively compact in $C(\overline{D}\cup \{\infty\})$. \end{proposition} \section{Proofs of main results} \begin{proof}[Proof of Theorem \ref{thm1.2}] Let $\lambda \in [ 0,\lambda _0)$ and $\mu \in [0,\mu _0)$. We define the sequences $(u_{k})_{k\geq 0}$ and $(v_{k})_{k\geq 0}$ by \begin{gather*} v_0 =\omega, \\ u_{k}= \theta - \lambda V_{m,n} ( p g(v_k)), \\ v_{k+1} =\omega - \mu V_{m,n} ( q f(u_k)) . \\ \end{gather*} We intend to prove that for all $k\in\mathbb{N}$, \begin{gather} 0 < ( 1- \frac{\lambda}{\lambda_0})\theta \leq u_{k} \leq u_{k+1}\leq \theta , \label{a1} \\ 0 < (1- \frac{\mu}{\mu_0})\omega \leq v_{k+1}\leq v_{k} \leq \omega. \label{a2} \end{gather} Note that from the definition of $\lambda _0$ and $\mu _0$ we have \begin{gather} \lambda _0 V_{m,n}(pg(\omega)) \leq \theta , \label{*1}\\ \mu _0V_{m,n}(qf(\theta)) \leq \omega . \label{**1} \end{gather} From \eqref{*1} we have \[ u_0=\theta -\lambda V_{m,n}(pg(v_0 ))\geq (1-\frac{\lambda }{\lambda _0})\theta >0. \] Then $v_{1}-v_0=-\mu V_{m,n}(qf(u_0))\leq 0$. Since $g$ is nondecreasing we obtain \[ u_{1}-u_0=\lambda V_{m,n}(p(g(v_0)-g(v_{1})))\geq 0. \] Now, since $v_0$ is positive and $f$ is nondecreasing, \[ v_{1}\geq \omega -\mu V_{m,n}(q\,f(\theta)). \] We deduce from \eqref{**1} that \[ v_{1}\geq (1-\frac{\mu }{\mu _0}) \omega >0. \] This implies that $u_{1}\leq \theta$. Finally, we obtain that \begin{gather*} 0 < ( 1- \frac{\lambda}{\lambda_0})\theta \leq u_0 \leq u_{1}\leq \theta,\\ 0 < (1- \frac{\mu}{\mu_0})\omega \leq v_{1}\leq v_0 \leq \omega. \end{gather*} By induction, we suppose that \eqref{a1} and \eqref{a2} hold for $k$. Since $f$ is nondecreasing and $u_{k+1}\leq \theta $, we have \[ v_{k+2}-v_{k+1}=\mu V_{m,n}(q(f(u_{k}) -f(u_{k+1})))\leq 0, \] and \begin{align*} v_{k+2} &=\omega -\mu V_{m,n}(q\,f(u_{k+1}))\\ &\geq \omega -\mu V_{m,n}(qf(\theta ))\\ &\geq (1-\frac{\mu }{\mu _0})\omega . \end{align*} To reach the last inequality, we use \eqref{**1}. Then \[ 0<(1-\frac{\mu }{\mu _0})\omega \leq v_{k+2}\leq v_{k+1}\leq \omega . \] Now, using that $g$ is nondecreasing we have \[ u_{k+2}-u_{k+1}=\lambda V_{m,n}(p(g(v_{k+1}) -g(v_{k+2}))\geq 0. \] Since $v_{k+2}>0$, we obtain \[ 0<(1-\frac{\lambda }{\lambda _0})\theta \leq u_{k+1}\leq u_{k+2}\leq \theta . \] Therefore, the sequences $(u_{k})_{k\geq 0}$ and $(v_{k})_{k\geq 0}$ converge respectively to two functions $u$ and $v$ satisfying \begin{gather*} (1-\frac{\lambda }{\lambda _0})\theta\leq u \leq \theta, \\ (1-\frac{\mu }{\mu _0})\omega\leq v \leq \omega .% \label{21} \end{gather*} % \label{21} We claim that \begin{gather} u=\theta -\lambda V_{m,n}(p\,g(v)) , \label{s23}\\ v= \omega-\mu V_{m,n}(q\,f(u)). \label{s'23} \end{gather} Since $v_{k}\leq \omega $ for all $k\in \mathbb{N} $, using hypothesis (H$_{3}$) and the fact that $g$ is nondecreasing, there exists $\tilde{p}\in K_{m,n}^{\infty }(D)$ such that \begin{equation} pg(v)\leq pg(\omega )\leq \tilde{p}\,\gamma, \label{c} \end{equation} and so $p|g(v_{k})-g(v)|\leq 2\tilde{p} \gamma$ for all $k\in\mathbb{N}$. From Proposition \ref{prop2.4}, we obtain \begin{equation} V_{m,n} (\tilde{p}\, \gamma)\in C(\overline{D}), \label{c'} \end{equation} and by Lebesgue's theorem we deduce that \[ \lim_{k\to \infty}V_{m,n}(pg(v_{k}))=V_{m,n}(pg(v)). \] So, letting $k\to \infty $ in the equation $u_{k}=\theta-\lambda V_{m,n}(pg(v_{k}))$, we obtain \eqref{s23}. Similarly, we obtain \eqref{s'23}. Next, we claim that $(u,v)$ satisfies \begin{equation} \begin{gathered} (-\Delta)^{m} u+\lambda p g(v)=0, \\ (-\Delta)^{m} v+\mu q f(u)=0. \end{gathered} \label{g} \end{equation} Indeed, using \eqref{c} and Proposition \ref{prop2.3}, we obtain $p g(v)\in L^1_{\rm loc}(D)$. Using again \eqref{c}, it follows from \eqref{c'} that \[ V_{m,n}(p g(v))\in C(\overline{D}). \] Which implies that \[ V_{m,n}(p g(v))\in L^1_{\rm loc}(D). \] Similarly \[ qf(u),\text{ }V_{m,n}(qf(u))\in L_{\rm loc}^{1}( D ). \] Now, applying the operator $(-\Delta)^{m} $ in both \eqref{s23} and \eqref{s'23}, we deduce that $(u,v)$ is a positive solution (in the sense of distributions) of \eqref{g}. On the other hand, using Proposition \ref{prop2.4} and \eqref{c}, we deduce that $$ x\to \frac{V_{m,n}(p g(v))(x)}{(|x|^2-1)^{m-1}}\in C_0(D) $$ and $$ x\to \frac{V_{m,n}(q f(u))(x)}{(|x|^2-1)^{m-1}}\in C_0(D). $$ Thus, we deduce from \eqref{s23} and \eqref{s'23} that \begin{gather*} \lim_{x\to\xi \in\partial D}\frac{u(x)}{(|x|^2-1)^{m-1}}=a\varphi(\xi), \quad \lim_{x\to\xi \in\partial D}\frac{v(x)}{(|x|^2-1)^{m-1}}=b\psi(\xi) , \\ \lim_{|x|\to \infty}\frac{u(x)}{(|x|^2-1)^{m-1}}=\alpha ,\quad \lim_{|x|\to \infty}\frac{v(x)}{(|x|^2-1)^{m-1}}=\beta . \end{gather*} % \label{1} Furthermore, the continuity of $\theta, \omega, V_{m,n}(p g(v))$ and $V_{m,n}(q f(u))$ imply that $(u,v)\in (C(D))^2$. This completes the proof. \end{proof} \begin{proof}[Proof of Theorem \ref{thm1.3}] Put $c=1+\alpha_{p_{1}}+ \alpha_{q_{1}}$, where $\alpha_{p_{1}}$ and $\alpha_{q_{1}}$ are the constants given in Proposition \ref{prop2.2} and associated respectively to the functions $p_1$ and $q_1$ given in hypothesis $(H_5)$. Suppose that $\varphi\geq c\phi$ and $\psi\geq c\phi$. Then it follows from the maximum principle that for each $x\in D$, we have \begin{gather} H_{D}\varphi (x)\geq c\,h_0(x), \label{***} \\ H_{D}\psi (x)\geq c\,h_0(x).\label{*} \end{gather} We consider the non-empty closed convex set \[ \Lambda =\{w\in C(\overline{D}\cup \{\infty\}) :h_0\leq w\leq H_{D}\varphi \}. \] We define the operator $T$ defined on $\Lambda$ as $$ Tw=H_{D}\varphi -\frac{V_{m,n}(pg[\gamma(\beta h+ H_{D}\psi)-V_{m,n}(qf(\tilde{w}) )]) }{\gamma}, $$ where $\tilde{w}(x)=\gamma(x)(w(x)+\alpha h(x))=(|x|^2-1)^{m-1}(w(x)+\alpha h(x))$. We need to check that the operator $T$ has a fixed point $w$ in $\Lambda$. First, we prove that $T\Lambda$ is relatively compact in $C(\overline{D}\cup \{\infty\})$. Let $w\in \Lambda$, then we have $w+\alpha h\geq h_0$. \\Since $f$ is nonincreasing, it follows from Proposition \ref{prop2.2} that \[ V_{m,n}(q f(\tilde{w})) \leq V_{m,n}(q f(\gamma\,h_0)) =V_{m,n}(q f(\rho_0)) \leq \alpha_{q_{1}}\, \rho_0. \] Which implies \begin{equation} \gamma (\beta h+ H_{D}\psi)- V_{m,n}(q f(\tilde{w}))\geq \gamma (\beta h+ H_{D}\psi-\alpha_{q_{1}}\,h_0). \label{R1} \end{equation} According to \eqref{*}, we obtain \begin{equation} \gamma (\beta h+ H_{D}\psi)- V_{m,n}(q f(\tilde{w}))\geq \gamma (\beta h+ h_0)\geq \rho_0. \label{R2} \end{equation} Hence \begin{equation} Tw \leq H_{D}\varphi. \label{oss1} \end{equation} Also, since $g$ is nonincreasing, we obtain \begin{equation} pg(\gamma (\beta h+ H_{D}\psi)-V_{m,n}(qf(\tilde{w}) ))\leq pg(\rho_0). \label{**} \end{equation} So it follows that for each $y\in D$, we have \begin{equation} \frac{p(y)g[\gamma(y)(\beta h(y)+ H_{D}\psi(y))-V_{m,n}(q f(\tilde{w}) )(y)]}{\gamma(y)}\leq p_1(y)\,h_0(y). \label{R4} \end{equation} Therefore, we deduce from (H5) and Proposition \ref{prop2.5} that the family of functions $$ \big\{x\to \frac{V_{m,n}(pg[\gamma(\beta h+ H_{D}\psi)-V_{m,n}(q f(\tilde{w}) )])(x)}{\gamma(x)},\;w\in \Lambda\big\} $$ is relatively compact in $C(\overline{D}\cup \{\infty\})$. Moreover, since $H_{D}\varphi\in C(\overline{D}\cup \{\infty\})$, we have the set $T\Lambda$ is relatively compact in $C(\overline{D}\cup \{\infty\})$. Next, we claim that $T\Lambda \subset\Lambda$. Indeed, let $\omega\in \Lambda$, by using \eqref{**}, $(H_5)$ and Proposition \ref{prop2.2}, we have \[ \frac{V_{m,n}(pg[\gamma (\beta h+ H_{D}\psi)-V_{m,n}(q f(\tilde{w}) )])(x)}{\gamma(x)}\leq \alpha_{p_{1}}\,h_0(x), \] for each $x\in D$. According to \eqref{***}, we obtain \[ Tw(x)\geq (1+\alpha_{q_{1}})h_0(x)\geq h_0(x),\quad \text{for each }x\in D. \] This, together with \eqref{oss1}, proves that $T\omega\in \Lambda$. Now, we prove the continuity of the operator $T$ in $\Lambda$ with respect to the supremum norm. Let $(w_{k})_{k\in \mathbb{N}}$ be a sequence in $\Lambda$ which converges uniformly to a function $w\in \Lambda$. Then, for each $x\in D$, we have \begin{equation} |Tw_{k}(x)-Tw(x)| \leq \frac{V_{m,n}(p|g(s_k)-g(s)|)(x)}{\gamma(x)} , \label{M2} \end{equation} where $s_k=\gamma(\beta h+ H_{D}\psi)-V_{m,n}(q f(\gamma(w_{k}+\alpha h)) )$ and $s=\gamma(\beta h+ H_{D}\psi)-V_{m,n}(q f(\gamma(w+\alpha h)) )$. Using the fact that $g$ is nonincreasing and $\eqref{R1}$ , we have \begin{align*} p(g(s_k)+g(s)) &\leq 2pg(\gamma(\beta h+ H_{D}\psi- \alpha_{q_{1}} h_0))\\ &\leq 2p g(\rho_0)=2p_1\rho_0 . \end{align*} To reach the last inequality we use \eqref{*}. Since from (H5) and Proposition \ref{prop2.4}, the function $$ x\to \int_D (\frac{|y|^2-1}{|x|^2-1})^{m-1}G_{m,n}^D (x,y)\,h_0(y)\,p_1(y)dy $$ is in $C_0(D)$, also using the fact that \[ p|g(s_k)-g(s)|\leq p(g(s_k)+g(s)), \] it follows from \eqref{M2} and the dominated convergence theorem that for each $x\in D$, the sequence $(Tw_k(x))$ converges to $Tw(x)$ as $k\to \infty$. Since $T\Lambda$ is relatively compact in $C(\overline{D}\cup \{\infty\})$, we deduce that the pointwise convergence implies the uniform convergence; that is, \[ \| Tw_{k}-Tw\|_{\infty }\to 0\quad\text{as }k\to \infty . \] This shows that $T$ is a continuous mapping from $\Lambda$ into itself. Then by using Schauder fixed point theorem, there exists $w\in \Lambda$ such that $Tw=w$. Now, for each $x\in D$, put \begin{gather} u(x)=(|x|^2-1)^{m-1}(w(x)+\alpha h(x)), \label{R7}\\ v(x)=(|x|^2-1)^{m-1}(\beta h(x)+H_D\psi(x))-V_{m,n}(qf(u))(x). \label{R8} \end{gather} Then \begin{equation} u(x)-\alpha(|x|^2-1)^{m-1} h(x)= (|x|^2-1)^{m-1}H_D\varphi(x)-V_{m,n}(p g(v))(x). \label{R5} \end{equation} As the remainder of the proof, we aim to show that $(u,v)$ is the desired solution of problem \eqref{Q}. By using respectively \eqref{R7}, \eqref{R8} and \eqref{R2}, clearly $(u,v)$ satisfies for each $x\in D$, \begin{equation} h_0(x)+ \alpha h(x)\leq\frac{u(x)}{(|x|^2-1)^{m-1}}\leq H_D\varphi(x)+ \alpha h(x) \label{R6} \end{equation} and \[ h_0(x)+ \beta h(x)\leq\frac{v(x)}{(|x|^2-1)^{m-1}}\leq H_D\psi(x)+ \beta h(x). \] On the other hand, from \eqref{R7}, we have $u(x)\geq \rho_0(x)$ for each $x\in D$. Since $f$ is nonincreasing, this implies \[ q f(u)\leq q f(\rho_0)=q_1\rho_0. \] Note that from (H5) we have $q_1$ is in the Kato class $K_{m,n}^{\infty }(D)$, so it follows from Proposition \ref{prop2.3} that $qf(u)\in L_{\rm loc}^1(D)$ and from Proposition \ref{prop2.2} that $V_{m,n}(q\,f(u))\in L_{\rm loc}^1(D)$. Similarly, we obtain $pg(v)\in L_{\rm loc}^1(D)$ and $V_{m,n}(pg(v))\in L_{\rm loc}^1(D)$. Then applying the elliptic operator $(-\Delta)^m$ in both \eqref{R7} and \eqref{R8}, we obtain clearly that $(u,v)$ is a positive continuous solution (in the distributional sense) of \begin{gather*} (-\Delta)^{m} u+ p(x)g(v)=0,\quad x\in D, \\ (-\Delta)^{m} v+ q(x)f(u)=0,\quad x\in D. \end{gather*} Finally, from \eqref{R5}, \eqref{R4}, Proposition \ref{prop2.4} and the fact that $H_D\varphi = \varphi$ on $\partial D$, we conclude that \[ \lim_{x\to\xi \in\partial D}\frac{u(x)}{(|x|^2-1)^{m-1}} =\varphi(\xi). \] Also, since $\lim_{|x|\to\infty }H_D\varphi(x)= \lim_{|x|\to\infty}h_0(x)=0$, it follows from \eqref{R6} that \[ \lim_{|x|\to\infty}\frac{u(x)}{(|x|^2-1)^{m-1}}=\alpha . \] The proof is complete by using the same arguments for $v$. \end{proof} \section{Examples} In this Section, we give some examples where hypotheses (H2) and (H3) are satisfied. \begin{example} \label{exmp4.1}\rm Let $\alpha=1$, $a=0$, $\beta=1$ and $b=0$. Let $f$ and $g$ be two nonnegative nondecreasing bounded continuous functions on $(0,\infty)$. Assume that $p$ and $q$ are two nonnegative measurable functions on $D$ satisfying $$ p(x)\leq \frac{1}{|x|^{\nu-\kappa}( |x|-1)^\kappa},\quad q(x)\leq \frac{1}{|x|^{\nu-\kappa}( |x|-1)^\kappa}, $$ with $\kappa2$. \end{example} Since $|x|+1\asymp |x|$, for each $x\in D$, then we have \begin{gather*} \frac{p(x)\,g(\omega(x))}{(|x|^2-1)^{m-1}}\leq \frac{C}{|x|^{\nu-\kappa + m-1}( |x|-1)^{m-1+\kappa}}, \\ \frac{q(x)\,f(\theta(x))}{(|x|^2-1)^{m-1}}\leq \frac{C}{|x|^{\nu-\kappa + m-1}( |x|-1)^{m-1+\kappa}}. \end{gather*} Using the fact that $\kappa2$, it follows that the functions \[ x\to \frac{p(x)\,g(\omega(x))}{(|x|^2-1)^{m-1}}\quad\text{and}\quad x\to \frac{q(x)\,f(\theta(x))}{(|x|^2-1)^{m-1}} \] are in $K_{m,n}^{\infty }(D)$. Now, since for each $x\in D$, we have \begin{gather} h(x)=1-\frac{1}{|x|^{n-2}}\asymp \frac{|x|-1}{|x|}, \label{S2}\\ \theta(x)= (|x|^2-1)^{m-1}h(x)=\omega(x), \notag \end{gather} then there exists $C>0$ such that \[ p(x)g(\omega(x))\leq \frac{C}{|x|^{\nu-\kappa + m-2}( |x|-1)^{m+\kappa}}\omega(x),\quad\text{for each }x\in D. \] So, we deduce from the choice of $\nu,\kappa$ that there exists $p_0\in K_{m,n}^{\infty }(D)$ such that \[ p(x) g(\omega(x))\leq p_0(x)\,\omega(x). \] Which implies from Proposition \ref{prop2.2} that $V_{m,n}(p\,g(\omega))\leq C\,\omega$. Hence $\lambda_0 > 0$. Similarly, we have $\mu_0 > 0$. \begin{example} \label{exmp4.2} \rm Let $\alpha=1$, $a=0$, $\beta=0 $ and $b=1$. Assume that $\psi \geq c_0 > 0$ on $\partial D$. Let $f$ and $g$ be two continuous and nondecreasing functions on $(0,\infty)$ satisfying for $t\in (0,\infty)$ \begin{equation} 0\leq g(t)\leq \eta \,t\;\text{and}\;0\leq f(t)\leq \xi\,t, \label{S5} \end{equation} where $\eta$ and $\xi$ are positive constants. Suppose furthermore that $p$ and $q$ are nonnegative measurable functions on $D$ such that $$ p(x)\leq \frac{1}{|x|^{\delta-\sigma}( |x|-1)^\sigma},\quad q(x)\leq \frac{1}{|x|^{s-r}( |x|-1)^r}, $$ where \begin{gather} \sigma+1<2m<\delta+n-2, \label{S3}\\ r-1<2m<2-n+s .\label{S4} \end{gather} Here $\theta(x)=(|x|^{2}-1)^{m-1} h(x)$ and $\omega(x)=(|x|^{2}-1)^{m-1}H_D\psi(x) $. \end{example} Since $\psi\geq c_0>0$, it follows that \begin{equation} H_D\psi(x)\asymp H_D1(x)=\frac{1}{|x|^{n-2}},\quad \text{for each }x\in D. \label{S1} \end{equation} Then, from \eqref{S5}, we have \begin{equation} \frac{p(x)\,g(\omega(x))}{(|x|^2-1)^{m-1}} \leq \eta p(x)H_D\psi(x) \leq\frac{C}{|x|^{n-2+\delta-\sigma}(|x|-1)^\sigma}.\label{S6} \end{equation} Also, using \eqref{S2}, we have \[ \frac{q(x) f(\theta(x))}{(|x|^2-1)^{m-1}} \leq \xi q(x) h(x) \leq \frac{C}{|x|^{1+s-r}(|x|-1)^{r-1}}. \] This, together with \eqref{S3}, \eqref{S4} and \eqref{S6}, implies that (H3) is satisfied. Now, using \eqref{S2}, \eqref{S5} and \eqref{S1}, for each $x\in D$, we have \[ p(x) g(\omega(x)) \leq \eta p(x) \omega(x) \leq C\,p(x) (|x|^2-1)^{m-1}H_D1(x) \leq \frac{C (|x|^2-1)^{m-1}\,h(x)}{|x|^{n-3+\delta-\sigma}(|x|-1) ^{\sigma+1}}. \] So it follows from \eqref{S3} that there exists $p_2\in K_{m,n}^{\infty }(D)$ such that $p\,g(\omega)\leq p_2\,\theta$. Hence, it follows from Proposition \ref{prop2.2} that $V_{m,n}(p\,g(\omega))\leq C\theta$, which implies that $\lambda_0 > 0$. Using again \eqref{S2}, we obtain, for each $x\in D$, \[ q(x)\,h(x) \leq \frac{C}{|x|^{1+s-r}(|x|-1)^{r-1}}. \] According to \eqref{S5} and \eqref{S4}, there exists $q_2 \in K_{m,n}^{\infty }(D)$ satisfying \[ qf(\theta)\leq C \gamma q_2 H_D1 . \] Finally, we deduce from \eqref{S1} and Proposition \ref{prop2.2} that $V_{m,n}(q\,f(\theta))\leq C\omega$. This implies that $\mu_0>0$. We end this section by giving an example as an application of Theorem \ref{thm1.3}. \begin{example}\label{examp4.3} \rm Let $\tau>0$, $\varepsilon>0$, $g(t)=t^{-\tau}$ and $f(t)=t^{-\varepsilon}$. Let $p$ and $q$ be two nonnegative measurable functions in $D$ satisfying \begin{gather*} p(x)\leq \frac{1}{( |x|-1)^{l-(1+\tau)m} |x|^{\vartheta-l+(1+\tau)(n-m)}},\\ q(x)\leq \frac{1}{( |x|-1)^{k-(1+\varepsilon)m} |x|^{\zeta-k+(1+\varepsilon)(n-m)}}, \end{gather*} where $l<2m<\vartheta$ and $k<2m<\zeta$. Let $\phi$ be a nonnegative nontrivial continuous function on $\partial D$ and put $\rho_0(x)=(|x|^2-1)^{m-1}H_D\phi(x)$ for $x\in D$. \end{example} Since for $x\in D$, we have \[ H_D\phi(x)\geq C\, \frac{|x|-1}{(|x|+1)^{n-1}}. \] Then we obtain for each $x\in D$ that \[ p_1(x)=p(x) \rho_0^{-\tau-1}(x)\leq \frac{C}{(|x|-1)^{l}|x|^{\vartheta-l}}. \] Similarly, we have \[ q_1(x) \leq \frac{C}{(|x|-1)^{k}|x|^{\zeta-k}},\quad x\in D. \] Hence, hypothesis (H5) is satisfied. So there exists $c>1$ such that if $\varphi$ and $\psi$ are two nonnegative nontrivial continuous functions on $\partial D$ satisfying $\varphi\geq c\phi$ and $\psi\geq c \phi$ on $\partial D$, then for each $\alpha \geq 0$ and $\beta \geq 0$, problem \begin{gather*} (-\Delta)^{m} u+ p(x)\,v^{-\tau}=0,\quad x\in\,D, \quad (\text{in the sense of distributions}), \\ (-\Delta)^{m} v+ q(x)\,u^{-\varepsilon}=0,\quad x\in\,D,\\ \lim_{x\to s \in\partial D}\frac{u(x)}{(|x|^2-1)^{m-1}}=\varphi(s) ,\quad \lim_{x\to s \in\partial D}\frac{v(x)}{(|x|^2-1)^{m-1}}=\psi(s) , \\ \lim_{|x|\to \infty}\frac{u(x)}{(|x|^2-1)^{m-1}}=\alpha ,\quad \lim_{|x|\to \infty}\frac{v(x)}{(|x|^2-1)^{m-1}}=\beta , \end{gather*} has a positive continuous solution $(u,v)$ satisfying for each $x\in D$, \begin{gather*} (|x|^2-1)^{m-1}(\alpha\,h(x)+h_0(x))\leq u(x) \leq (|x|^2-1)^{m-1}(\alpha\,h(x)+H_D\varphi(x)),\\ (|x|^2-1)^{m-1}(\beta\,h(x)+h_0(x)) \leq v(x) \leq (|x|^2-1)^{m-1}(\beta\,h(x)+H_D\psi(x)). \end{gather*} \subsection*{Acknowledgments} I am grateful to professor Habib M\^aagli for his guidance and useful discussions. I also want to thank the anoymous referee for the careful reading of this article. \begin{thebibliography}{00} \bibitem{Armitage} David H. Armitage and Stephen J. Gardiner; \emph{Classical potential theory}, Springer 2001. \bibitem{Athr} S. Athreya; \emph{On a singular semilinear elliptic boundary value problem and the boundary Harnack principle}, Potential Anal. 17 (2002), 293-301. \bibitem{BMZm1} I. Bachar, H. M\^{a}agli, N. Zeddini; \emph{Estimates on the Green function and existence of positive solutions of nonlinear singular elliptic equations}, Commun. Contemp. Math. 5 (3) (2003), 401-434. \bibitem{BMZ} I. Bachar, H. M\^{a}agli, N. Zeddini; \emph{Estimates on the Green function and existence of positive solutions for some nonlinear polyharmonic problems outside the unit ball}, Anal. Appl. 6 (2) (2008) 121-150. \bibitem{Boggio} T. Boggio; \emph{Sulle funzioni di Green d'ordine m}, Rend. Circ. Math. Palermo, 20 (1905) 97-135. \bibitem{cirstea} F. C. C\^{i}rstea, V. D. Radulescu; \emph{Entire solutions blowing up at infinity for semilinear elliptic systems}, J. Math. Pures. Appl. 81 (2002) 827-846. \bibitem{david89} F. David; \emph{Radial solutions of an elliptic system}, Houston J. Math. 15 (1989) 425-458. \bibitem{Garabedian} P. R. Garabedian; \emph{A partial differential equation arising in conformal mapping}, Pacific J. Math. 1 (1951) 485-524. \bibitem{ghanmi} A. Ghanmi, H. M\^{a}agli, V. D. Radulescu, N. Zeddini; \emph{Large and bounded solutions for a class of nonlinear Schr\"{o}dinger stationary systems}, Analysis and Applications 7 (2009) 391-404. \bibitem{GMTZ} A. Ghanmi, H. M\^{a}agli, S. Turki, N. Zeddini; \emph{Existence of positive bounded solutions for some nonlinear elliptic systems}. J. Math. Anal. Appl. 352 (2009) 440-448. \bibitem{ghergu1} M. Ghergu, V. D. Radulescu; \emph{On a class of singular Gierer-Meinhart systems arising in morphogenesis}, C. R. Acad. Sci. Paris. Ser.I 344 (2007) 163-168. \bibitem{ghergu2} M. Ghergu, V. D. Radulescu; \emph{A singular Gierer-Meinhardt system with different source terms, Proceedings of the Royal Society of Edinburgh}, Section A (Mathematics) 138A (2008) 1215-1234. \bibitem{ghergu3} M. Ghergu, V. D. Radulescu; \emph{Singular elliptic problems. Bifurcation and Asymptotic Analysis}, Oxford Lecture Series in Mathematics and Its Applications, Vol. 37, Oxford University Press, 2008. \bibitem{lairwood} A. V. Lair, A. W. Wood; \emph{Existence of entire large positive solutions of semilinear elliptic systems}, Journal of Differential Equations 164. No.2 (2000) 380-394. \bibitem{zhou} D. Ye, F. Zhou; \emph{Existence and nonexistence of entire large solutions for some semilinear elliptic equations}, J. Partial Differential Equations 21 (2008) 253-262. \end{thebibliography} \end{document}