\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2012 (2012), No. 115, pp. 1--14.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2012 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2012/115\hfil Existence of solutions]
{Existence of solutions for Hardy-Sobolev-Maz'ya systems}

\author[J. Wang, X. Wei \hfil EJDE-2012/115\hfilneg]
{Jian Wang, Xin Wei}  % in alphabetical order

\address{Jian Wang \newline
 Department of Mathematics,
 Jiangxi Normal University\\
 Nanchang, Jiangxi 330022, China}
\email{jianwang2007@126.com}

\address{Xin Wei \newline
Department of Finance, 
Jiangxi Normal University\\
Nanchang, Jiangxi 330022, China}
\email{wxgrat@sohu.com}

\thanks{Submitted December 26, 2011. Published July 5, 2012.}
\subjclass[2000]{35J47, 35J50, 35J57, 58E05}
\keywords{Variational identity; (PS) condition; linking theorem;
\hfill\break\indent  Hardy-Sobolev-Maz'ya inequality}

\begin{abstract}
 The main goal of  this article is to investigate the existence
 of solutions for the Hardy-Sobolev-Maz'ya system
 \begin{gather*}
 -\Delta u-\lambda \frac{u}{|y|^2}=\frac{|v|^{p_t-1}}{|y|^t}v,\quad
 \text{in }\Omega,\\
 -\Delta v-\lambda \frac{v}{|y|^2}=\frac{|u|^{p_s-1}}{|y|^s}u,\quad
 \text{in }\Omega,\\
 u=v=0,\quad  \text{on }\partial \Omega
\end{gather*}
 where $0\in\Omega$ which is a bounded, open and smooth subset of
 $\mathbb{R}^k\times \mathbb{R}^{N-k}$, $2\leq k<N$.
 The non-existence of classical positive solutions is
 obtained by a variational identity and the existence result
 by a linking theorem.
\end{abstract}

\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{definition}[theorem]{Definition}

\allowdisplaybreaks

\section{Introduction}

In this article, we are concerned with the existence of nontrivial
solutions for Hardy-Sobolev-Maz'ya  system
\begin{equation}\label{eq:1.1}
\begin{gathered}
-\Delta u-\lambda \frac{u}{|y|^2}=\frac{|v|^{p_t-1}}{|y|^t}v,\quad 
\text{in }\Omega,\\
-\Delta v-\lambda \frac{v}{|y|^2}=\frac{|u|^{p_s-1}}{|y|^s}u,\quad 
\text{in }\Omega,\\
u=v=0,\quad  \text{on }\partial \Omega,
\end{gathered}
\end{equation}
where $0\in\Omega$ which is an open, bounded and smooth domain of
$\mathbb{R}^N=\mathbb{R}^k\times \mathbb{R}^{N-k}$ with $2\leq k<N$.
A point $x\in \mathbb{R}^N$ is denoted as 
$x=(y,z)\in \mathbb{R}^k\times \mathbb{R}^{N-k}$. We also give some assumptions
for the parameters: $ 0\leq\lambda<\frac{(k-2)^2}{4}$ when $k>2$,
$\lambda=0$ when $k=2$, $0\leq t,s <2$ and $p_t,p_s>1$.

The Hardy-Sobolev-Maz'ya  elliptic equation:
\begin{equation}\label{2}
\begin{gathered}
-\Delta u-\lambda \frac{u}{|y|^2}=\frac{|u|^{p_t-1}}{|y|^t}u,\quad \text{in } \Omega,\\
u=0,\quad  \text{on }\partial \Omega,
\end{gathered}
\end{equation}
 comes from an astrophysics model with
$\Omega=\mathbb{R}^3$, $\lambda=0$, $t=1$, (see \cite{BG} for
details).
 The existence and regularity of the solution for
problem \eqref{2} in bounded domain have been studied in \cite{MK}
in subcritical case, that is, $p_t+1<\frac{2N-2t}{N-2}:=2^*(t)$, and
non-existence in super critical was obtained by Pohoza\u{e}v
identity. It is interesting to investigate with what restrictions on
$p_t,p_s$ for existence solutions for system \eqref{eq:1.1} by more
general variational identity, see \cite{V} for details.
 The natural functional  corresponding to system \eqref{eq:1.1} is
 \begin{eqnarray*}
 I_0(u,v)=\int_\Omega\Big(\nabla u\cdot\nabla
v-\lambda\frac{uv}{|y|^2}-\frac
1{p_t+1}\frac{|v|^{p_t+1}}{|y|^t}-\frac
1{p_s+1}\frac{|u|^{p_s+1}}{|y|^s}\Big)dx
\end{eqnarray*}
in the space $H^1_0(\Omega)\times H^1_0(\Omega)$ with natural exponent
region:
\begin{equation}\label{0.4}
p_t+1<2^*(t)\quad \rm{and} \quad  p_s+1<2^*(s).
\end{equation} The
quadratic part of the functional $I_0$, that is, $\int_\Omega(\nabla
u\cdot\nabla v-\lambda\frac{uv}{|y|^2})dx$, is positive on the
infinite dimensional subspace $\{(u,u)\in H^1_0(\Omega)\times
H^1_0(\Omega)\}$ and negative on the infinite dimensional subspace
$\{(u,-u)\in H^1_0(\Omega)\times H^1_0(\Omega)\}$. The system is
then called strongly indefinite.

There were a significant amount of research on strongly indefinite
elliptic systems, see \cite{F,V,JR,C}. In particular, in
\cite{F,LY}, the authors did the existence solutions for the
strongly indefinite elliptic systems with the weights.
 They extended the restriction of the exponent by destroying symmetry of
the regularity of solution pair, then obtained the existence results
by the linking type theorem.
  Inspired works in \cite{V,LY,MK,F}, we study that the
  existence of infinitely many solutions for system \eqref{eq:1.1} with
$$
\frac{1}{p_t+1}(1-\frac{t}{N})+\frac{1}{p_s+1}(1-\frac{s}{N})>\frac{N-2}{N}.
$$
which contains natural  exponent region \eqref{0.4}. It could happen
that the exponent $p_t$ or $p_s$ is supercritical in the sense that
$$
p_t+1>2^*(t) \quad\text{or}\quad  p_s+1>2^*(s),
$$
where the critical exponent $2^*(s)$ is from the the imbedding from
Sobolev space $H^1_0(\Omega)$ to
$$
L^{p_s+1}_s(\Omega)=\{u:\int_\Omega
\frac{|u|^{p_s+1}}{|y|^s}dx<+\infty\},
$$
 which is compact if $ 2\leq
p_s+1<2^*(s)$.  The main point to solve the problem  is to destroy
the symmetry between $u$ and $v$ by distributing more regularity of
$u$ than that of $v$ if $p_s\geq p_t$. To this end, we define
$A^{r}:=(-\Delta-\frac{\lambda}{|y|^2})^{r/2}$, which a positive
operator in  a fractional Sobolev space $E^r(\Omega):= D(A^r)$. Then
it is available to define the functional associated with system
\eqref{eq:1.1},
\begin{equation} \label{4}
I(u,v)=\int_\Omega\Big(A^ruA^{2-r}v-\frac{1}{p_t+1}
\frac{|v|^{p_t+1}}{|y|^t}-\frac{1}{p_s+1}\frac{|u|^{p_s+1}}{|y|^s}\Big)dx,
\end{equation}
in the fractional Sobolev space $E(\Omega)=E^r(\Omega)\times
E^{2-r}(\Omega)$.   The functional $I$ has critical points by using
linking type theorem (see \cite{LE}) in fractional Sobolev spaces
$E(\Omega)$. We have then the following existence results.

\begin{theorem}\label{th1.1}
Assume that $ 0\leq\lambda<\frac{(k-2)^2}{4}$ if $k>2$,
$\lambda=0$ if $k=2$, $0\leq t,s <2$ and $p_t,p_s>1$ satisfying
\begin{equation}\label{1}
\frac{1}{p_t+1}(1-\frac{t}{N})+\frac{1}{p_s+1}(1-\frac{s}{N})>\frac{N-2}{N},
\end{equation}
 then there are infinitely many
solutions  of system \eqref{eq:1.1}.

Moreover, we suppose that $\Omega$ is star-sharped with respect to
the origin and assumption \eqref{1} fails. Then system
\eqref{eq:1.1} does not have classical positive solution.
\end{theorem}


\begin{remark} \label{rmk} \rm
Under the assumption  \eqref{1}, we do not obtain a positive strong
solution of \eqref{eq:1.1}. Since it lacks regularity results for
\eqref{2} from weak sense to classical sense, then we can't use
Maximum Principle. For the regularity results, see \cite{MK} for
details.
\end{remark}

We observe that there are not just one singular point for weight
functions in system \eqref{eq:1.1}, but a manifold
$\{(0,z)\in\Omega\}$ with dimension $N-k$. We would like to
emphasize that the restriction hyperbola \eqref{1} does not depend
on the dimension number $k$, one reason for which is the critical
exponent of imbedding from $E^r(\Omega)\hookrightarrow
L^{p_s+1}_s(\Omega)$ independent of $k$. To be more precise,
$2^*(s)$ defined before is equal to the critical exponent of the
imbedding
$$
H^1_0(\Omega)\hookrightarrow
L^{p}(\Omega,\frac1{|x|^s}):=\big\{u:\int_\Omega
\frac{|u|^{p}}{|x|^s}dx<+\infty\big\}.
$$

This paper is organized as follows.  Section \S 2 is devoted to
study the compact imbedding from fractional Sobolev spaces to
weighted spaces. In Section \S 3 we prove the existence of
infinitely many solutions of \eqref{eq:1.1}. Finally, we do the
nonexistence result in Theorem \ref{th1.1} by variational identity
in Section \S 4.


\section{Compactness of fractional Sobolev space}

To destroy the symmetry of regularities between $u$ and
$v$, it is necessary to establish compact imbedding from fractional
Sobolev spaces to the weighted spaces
$$
L^{p_s+1}_s(\Omega)=\{u:\int_\Omega
\frac{|u(x)|^{p_s+1}}{|y|^s}dx<+\infty,
x=(y,z)\in\mathbb{R}^k\times\mathbb{R}^{N-k}\}.
$$

Firstly, we introduce interpolation theorem  (see \cite{LY,P}).
A pair $E_0,E_1$ of Banach spaces is called an interpolation pair, 
if $E_0$ and $E_1$ are continuously imbedded in some separated topological 
linear spaces $\mathcal{B}$. Let $A_0,\ A_1$ and $E_0,E_1$ be interpolation
pairs, $A_\theta$ and $E_\theta$ are called interpolation spaces of exponent 
$\theta (0<\theta<1)$, with respect
to $A_0,\ A_1$ and $E_0,\ E_1$ if we have the topological inclusions
$$
A_0\cap A_1\subset A_\theta\subset A_0+A_1,\quad
E_0\cap E_1\subset E_\theta\subset E_0+E_1,
$$
and if each linear mapping $T$ from a separated topological linear space 
$\mathcal{A}$ into  $\mathcal{B}$, which maps $A_i$ continuously into
$E_i$  $(i=0,1)$ and maps $A_\theta$
continuously into $E_\theta$ in such a way that
$$
M\leq M_0^{1-\theta}M_1^\theta,
$$
where $M$ denotes the norm of $T:A_\theta\to E_\theta$ and $M_i$ the norm of
$T:A_i \to E_i(i=0,1)$.

Let $E_0,E_1$ be the interpolation pairs. It requires the following condition
\cite{LY,P}
\begin{itemize}
\item[(H1)] For each compact set $\mathcal{K}\in E_0$ there exist a constant $C>0$ and $\mathcal{D}$
 of linear operators $P:\mathcal{B}\to\mathcal{B}$, which map $E_i$ into $E_0\cap E_1(i=0,1)$
 and such that
 \begin{equation}\label{eq:4.15}
 \|P\|_{L(E_i,E_i)}\leq C \quad (i=0,1).
  \end{equation}
Furthermore, we suppose that to each $\epsilon>0$ we can find a
$P\in \mathcal{D}$ such that
\begin{equation}\label{eq:4.16}
 \|Px-x\|_{E_0}<\epsilon
  \end{equation}
for all $x\in \mathcal{K}$.
\end{itemize}
  Stronger hypothesis of (H1) is the following.
\begin{itemize}
\item[(H2)] There exist a constant $C>0$ and a set $\mathcal{D}$ of linear operators
 $P:\mathcal{B}\to\mathcal{B}$ with $P(E_i)\subset E_0\cap E_1(i=0,1)$,
such that  \eqref{eq:4.15} is satisfied and every $\epsilon>0$ and every 
finite set $x_1,\dots,x_m$
 in $E_0$ we can find a $P\in \mathcal{D}$ so that
 \begin{equation}\label{eq:4.17}
 \|Px_k-x_k\|_{E_0}\leq\epsilon \quad (k=1,\dots,m).
  \end{equation}
\end{itemize}

 \begin{lemma}[\cite{P}] \label{lemma4.4}
Let $A_0,A_1$ and $E_0,E_1$ be interpolation pairs and suppose that $A_\theta$
and $E_\theta$   are interpolation spaces of exponent $\theta(0<\theta<1)$ with 
respect to these pairs.
Suppose further that $A_\theta\subset \overline{A_\theta}$ and $E_0,E_1$ satisfy
{\rm (H1)}.  Then, if $T:A_0\to E_0$ is compact and $T:A_1\to E_1$ is bounded, it
  follows that $T:A_\theta\to E_\theta$ is compact.
 \end{lemma}

To establish suitable interpolation pairs, we define the
fractional Sobolev space 
$E^r(\Omega):= D((-\Delta-\frac{\lambda}{|y|^2})^{r/2})$ with
$0\leq r\leq2$ which is a Hilbert space endowed with the norm
$\|u\|^2_{E^r}=\int_\Omega|A^ru|^2dx$,  induced by the inner product
$$
\langle u,v\rangle_{E^r}=\int_\Omega A^ruA^rvdx,
$$
where $A^r=(-\Delta-\frac{\lambda}{|y|^2})^{r/2}$.

Now we assume $0\leq r\leq2$, and define the interpolation spaces
$$
E^r(\Omega)=[H^2(\Omega)\cap H^1_0(\Omega),L^2(\Omega)]_{1-r}.
$$
In fact,
$$
-\Delta-\frac{\lambda}{|y|^2}: H^2(\Omega)\cap
H^1_0(\Omega)\subset L^2(\Omega)\to L^2(\Omega)
$$ 
and $D(-\Delta-\frac{\lambda}{|y|^2})=D(-\Delta)$. We have the following
spaces: 
$E^s=H^s(\Omega)$ if $0\leq s <1/2$; $E^{s}\subset
H^{s}(\Omega)$ if $ s =1/2$;
$E^s=\{u\in H^s(\Omega):u(x)=0, x\in\partial\Omega\}$ if $1/2<s \leq 2$,
 $s\not=3/2$; and
$E^{s}\subset\{u\in H^{s}(\Omega):u(x)=0,\ x\in\partial\Omega\}$ if
$s=3/2$.  See \cite{LE} for details.

Before using Lemma \ref{lemma4.4} to obtain the imbedding
$E^r(\Omega)\hookrightarrow L^{p_s+1}_s(\Omega)$, we first prove
some basic property of interpolation pair $L^{p_t+1}_t(\Omega)$ and
$L^2(\Omega)$ as follows.

\begin{proposition}\label{prop2.2}
The interpolation pair $L^{p_t+1}_t(\Omega),L^2(\Omega)$ satisfies
the condition {\rm (H2)}.
\end{proposition}

\begin{proof} 
 Let $f_1,\dots,f_m$ be given functions in $L^{p_t+1}_t(\Omega)$ and we
know $L^{p_t+1}_t(\Omega)\hookrightarrow L^2(\Omega)$.

 Suppose $\epsilon>0$ is a given number. As the set $\mathcal{E}$ of all 
bounded measurable functions with compact support is dense in $L^2(\Omega)$,
and then in $L^{p_t+1}_t(\Omega)$, we may assume that
$f_j\in \mathcal{E}(j=1,\dots,m)$.
 Let $K$ be a compact set in $\Omega$, outside of which all $f_j$ vanish,
and choose $\eta>0$ such that
 $\eta \max(1,\mu(K))<\epsilon$, where $\mu(K)$ is the Lebesgue measure of $K$. 
We  may construct  finite cubes 
$\{K_n:=K^y_n\times K^z_n\subset \mathbb{R}^k\times\mathbb{R}^{N-k}\}$  with  
$\mu  (K_n)>0$ and $K_0$ of measure zero such that
 $\sup_{ x,x'\in K_n}|f_j(x)-f_j(x')|<\eta \  (j=1,\dots,m)$ and  the union of 
$K_n$ including $n=0$  covers $K$.

 Let $\varphi_n$ $(n=1,2,\dots)$ denote the characterisitic function of $K_n$ 
and set
 $$
Pf:=\sum_{n=1}(\mu(K_n)^{-1}\int_\Omega f\varphi_ndx)\varphi_n,\quad
\text{for all }f\in L^{p_t+1}_t(\Omega).
$$
 We claim that \eqref{eq:4.15} holds for operator $P$. Indeed, 
by H\"{o}lder's inequality,
 \begin{align*}
&\int_\Omega\frac{|Pf|^{p_t+1}}{|y|^t}dx\\
&=\sum_{n=1}\Big(\mu(K_n)^{-1}\int_\Omega
f\varphi_ndx\Big)^{p_t+1}\int_{K_n}\frac{\varphi_n^{p_t+1}}{|y|^t}dx
\\
&\leq \sum_{n=1}\Big[\mu(K_n)^{-1}\Big(\int_{K_n}\frac{|f|^{p_t+1}}{|y|^t}dx\Big)
^{\frac{1}{p_t+1}}
\Big(\int_{K_n}|y|^\frac{t}{p_t}dx\Big)^{\frac{p_t}{p_t+1}}\Big]^{p_t+1}
\int_{K_n}\frac{1}{|y|^t}dx
\\
&\leq \sum_{n=1}\Big[\mu(K_n)^{-1}\int_{K_n}{|y|^t}dx\Big]
\Big[\mu(K_n)^{-1}\int_{K_n}\frac{1}{|y|^t}dx\Big]
 \int_{K_n}\frac{|f|^{p_t+1}}{|y|^t}dx
\\
&=\sum_{n=1}{\Big[\mu(K^y_n)^{-1}\int_{K^y_n}{|y|^t}dy\Big]
\Big[\mu(K^y_n)^{-1}\int_{K^y_n}\frac{1}{|y|^t}dy\Big]}
\int_{K_n}\frac{|f|^{p_t+1}}{|y|^t}dx
\end{align*}
The above equality uses $\mu(K_n)=\mu(K^y_n)\mu(K^z_n)$ and
$\int_{K_n}{|y|^t}dx=\mu(K^z_n)\int_{K^y_n}{|y|^t}dy$.

So we need only  prove
\begin{equation} \label{eq:4.21}
 {\Big[\mu(K^y_n)^{-1}\int_{K^y_n}{|y|^t}dy\Big]
\Big[\mu(K^y_n)^{-1}\int_{K^y_n}\frac{1}{|y|^t}dy\Big]}\leq C.
\end{equation}
where $C>0$ is independent of $n$.

In fact, for $\mu(K^y_n)>0$,  there is $\delta_n>0$ such that
$\mu(K^y_n)=\mu(B_{\delta_n}(0))$, where 
$B_{\delta_n}(0)\subset \mathbb{R}^k$. Since $K^y_n$ is cube, if
 $K^y_n\cap B_{\delta_n}(0)\not=\emptyset$, then
$$
\int_{K^y_n\cap
B_{\delta_n}}|y|^tdy\leq\delta_n^t\int_{B_{\delta_n}}dy=\delta_n^t\mu(K^y_n)
$$
and 
$$
\int_{K^y_n\cap B^c_{\delta_n}}|y|^tdy\leq
(c\delta_n)^t\int_{K^y_n\cap B^c_{\delta_n}}dy\leq
c^t\delta_n^t\mu(K^y_n),
$$
 where $c:=\sqrt{2}\mu(B_1)^{1/k}+1$ with $B_1$ being unit ball of
$\mathbb{R}^k$, which imply that 
$$
\int_{K^y_n}|y|^tdy\leq(c^t+1){\delta_n^t}\mu(K^y_n).
$$
On the other side, there is $C>0$ independent of $n$ such that
$$
\int_{K^y_n\cap
B_{\delta_n}}|y|^{-t}dy\leq\int_{B_{\delta_n}}|y|^{-t}dy=\frac{C}{k-t}{\delta_n^{-t}}\mu(K^y_n)
$$
and 
$$
\int_{K^y_n\cap B^c_{\delta_n}}|y|^{-t}dy\leq\delta_n^{-t}\int_{K^y_n\cap
B^c_{\delta_n}}dy\leq\delta_n^{-t}\mu(K^y_n),
$$
which imply that 
$$
\int_{K^y_n}|y|^{-t}dy\leq\frac{C+k-t}{k-t}\delta_n^{-t}\mu(K^y_n).
$$
Then we have
\begin{equation} \label{eq:4.22}
{\Big[\mu(K^y_n)^{-1}\int_{K^y_n}{|y|^t}dy\Big]
\Big[\mu(K^y_n)^{-1}\int_{K^y_n}\frac{1}{|y|^t}dy\Big]}
\leq C.
\end{equation}
for some $C>0$ independent of $n$.

 If $K^y_n\cap B_{\delta_n}(0)=\emptyset$, then we have
$r_n:=dist(0,K^y_n)\geq \delta_n$ and
$$
\int_{K^y_n}|y|^tdy\leq(r_n+c\delta_n)^t\mu(K^y_n),\ \int_{K^y_n}|y|^{-t}dy
\leq(r_n)^{-t}\mu(K^y_n),
$$
which imply
\begin{eqnarray}\label{eq:4.23}
 {[\mu(K^y_n)^{-1}\int_{K^y_n}{|y|^t}dy][\mu(K^y_n)^{-1}\int_{K^y_n}\frac{1}{|y|^t}dy]}\leq (\frac{r_n+c\delta_n}{r_n})^t\leq (1+c)^t.
\end{eqnarray}
Then \eqref{eq:4.21} follows from \eqref{eq:4.22} and
\eqref{eq:4.23}. Thus
$$
\int_\Omega\frac{|Pf|^{p_t+1}}{|y|^t}dx
\leq C\sum_{n=1}\int_{K_n}\frac{|f|^{p_t+1}}{|y|^t}dx =C
\int_{\Omega}\frac{|f|^{p_t+1}}{|y|^t}dx;
$$ 
that is,
$$
\|Pf\|_{L^{p_t+1}_t(\Omega)}\leq C\|f\|_{L^{p_t+1}_t(\Omega)}.
$$
Especially, setting $t=0$ and $p_t=1$, we have \eqref{eq:4.15}. Thus
the claim follows.
Next, we verify \eqref{eq:4.17}. Indeed,
\begin{align*}
 Pf_j(x)-f_j(x)
&=\sum_{n=1}[\mu(K_n)^{-1}\int_\Omega f_j(x')\varphi_n(x')dx']
 \varphi_n(x)-f_j(x) \\
&=\sum_{n=1}[\mu(K_n)^{-1}\int_\Omega
 (f_j(x')-f_j(x))\varphi_n(x')dx']\varphi_n(x)
 \end{align*}
and
$$
 |\mu(K_n)^{-1}\int_\Omega (f_j(x')-f_j(x))\varphi_n(x')dx'|\leq\eta,\ x\in {K_n}.
$$
It follows that
\begin{align*}
\|Pf_j-f_j\|_{L^{p_t+1}_t(\Omega)}&=\|Pf_j-f_j\|_{L^{p_t+1}_t(K)}
 \\
&\leq \sum_{n=1}[\mu(K_n)^{-1}\int_\Omega (f_j(x')-f_j(x))\varphi_n(x')dx']\varphi_n(x)
 \\
& \leq \eta \sum_{n=1}\mu(K_n)=\eta \mu(K)<\epsilon;
\end{align*}
 i.e., \eqref{eq:4.17} holds.
 The proof  is complete.
\end{proof}

Now we give the general imbedding theorem by the interpolation Lemma
\ref{lemma4.4}.

\begin{theorem}\label{th4.2}
The imbedding $E^r(\Omega)\hookrightarrow L^{p_s+1}_s(\Omega)$ is
 is compact if $2\leq {p_s+1} <\frac{2N-2s}{N-2r}$.
\end{theorem}

\begin{proof}
 We define the interpolation space,
$$
L^{q}_s(\Omega)=[L^{p_t+1}_t(\Omega),L^2(\Omega)]_{1-r}.
$$ 
We claim next  that $2\leq q \leq \frac{2N-2s}{N-2r}$.
In fact,  for any $u\in L^{p_t+1}_t(\Omega)$, by using H\"{o}lder's
inequality, one obtains
$$
\int_\Omega\frac{|u|^{q}}{|y|^s}dx
=\int_\Omega\frac{|u|^{2\gamma+(p_t+1)(1-\gamma)}}{|y|^s}dx
\leq \Big(\int_\Omega|u|^2dx\Big)^{\gamma}
\Big(\int_\Omega\frac{|u|^{p_t+1}}{|y|^{\frac{s}{1-\gamma}}}dx\Big)^{1-\gamma},
$$
where $\gamma=\frac{p_t+1-q}{p_t-1}\in(0,1)$.

Let $\theta=\frac{2\gamma}q$, then
$\frac{(p_t+1)(1-\gamma)}{q}=1-\theta$ and
$$
\Big(\int_\Omega\frac{|u|^{q}}{|y|^s}dx\Big)^\frac{1}{q}
 \leq \|u\|_{L^2(\Omega)}^{\theta}
\Big(\int_\Omega\frac{|u|^{p_t+1}}{|y|^\frac{s}{1-\gamma}}dx
 \Big)^{\frac{1-\theta}{p_t+1}}
=\|u\|_{L^2(\Omega)}^{\theta}
\Big(\int_\Omega\frac{|u|^{p_t+1}}{|y|^\frac{2s}{2-q(1-r)}}dx
\Big)^{\frac{1-\theta}{p_t+1}},
$$
where $ r=1-\theta$. The critical exponent of
$\int_\Omega\frac{|u|^{p_t+1}}{|y|^\frac{2s}{2-q(1-r)}}dx$ is
$(p_t+1)^*(r)=\frac{2(N-\frac{2s}{2-q(1-r)})}{N-2}$. Requiring
$p_t+1\leq (p_t+1)^*(r)$, we obtain
$$
2\leq q\leq\frac{2N-2s}{N-2r}.
$$
Hence, the claim is true.

By Proposition \ref{prop2.2}, we know that interpolation pair
$L^{p_t+1}_t(\Omega),\ L^2(\Omega)$  has  property
(H2). And the imbedding
$$
H^1_0(\Omega)\hookrightarrow  L^{p_s+1}_s(\Omega)
$$  
is compact if $2\leq p_s+1<2^*(s)$. Then by Lemma \ref{lemma4.4}, 
we obtain the results.
\end{proof}

Similarly, we have  $E^{2-r}(\Omega) \hookrightarrow
L^{p_t+1}_{t}(\Omega), \ \rm{if} \ 2\leq
{p_t}+1\leq\frac{2N-2t}{N+2r-4}$. Hence we obtain the following
theorem.

\begin{theorem}\label{th4.3}
The imbedding of the fractional Sobolev space
$$ 
E(\Omega)=E^r(\Omega)\times E^{2-r}(\Omega)\hookrightarrow 
 L^{p_s+1}_s(\Omega)\times L^{p_t+1}_t(\Omega)
$$
is compact, where $2\leq p_s+1< \frac{2N-2s}{N-2r}$, 
$2\leq p_t+1<\frac{2N-2t}{N+2r-4}$
 and $0<r<2$.
\end{theorem}


\begin{remark} \label{remark1}  \rm
If $2\leq p_s+1< \frac{2N-2s}{N-2r}, 2\leq
p_t+1<\frac{2N-2t}{N+2r-4}$, where $0<r<2$, then \eqref{1} holds.
Conversely, for $p_s, p_t>1$ satisfying \eqref{1}, then there exists
$r\in(0,2)$ such that $2\leq p_s+1< \frac{2N-2s}{N-2r}, 2\leq
p_t+1<\frac{2N-2t}{N+2r-4}$.
\end{remark}

\begin{lemma}\label{eigenvalue}
Suppose that $\Omega$ is an open, smooth and bounded domain and
$\lambda\in[0,(k-2)^2/4)$. Then there exists a sequence eigenvalues
$(\mu_n)_n$ and corresponding eigenfunctions $(\varphi_n)_n$ of
\begin{equation}
\begin{gathered}
-\Delta u-\lambda \frac{u}{|y|^2}=\mu u,\\
u\in H^1_0(\Omega)
\end{gathered}
\end{equation}
such that
$0<\mu_1<\mu_2\le\dots\le\mu_n\dots\to +\infty$
as $n\to+\infty$,
$\|\varphi_n\|_{H^1_0(\Omega)}=\mu_n\|\varphi_n\|_{L^2(\Omega)}$,
where $\|\varphi_n\|^2_{H^1_0(\Omega)}=\int_{\Omega}(|\nabla
\varphi_n|^2-\lambda \frac{\varphi_n^2}{|y|^2})dx$.
\end{lemma}

\begin{proof}
 Since $\lambda\in[0,(k-2)^2/4)$, then the norm
$(\int_{\Omega}|\nabla u|^2dx)^{1/2}$ is equivalent to
$[\int_{\Omega}(|\nabla u|^2-\lambda \frac{u^2}{|y|^2})dx]^2$ in
Hilbert space $H^1_0(\Omega)$. We observe that the operator
$S=(-\Delta- \frac{\lambda}{|y|^2})^{-1}$ is symmetric and compact,
following the the proceeding the proof of Theorem 2 in Chapter\S 6.5
in \cite{E}, we will have the results.
\end{proof}

We end this section with the fact that
$H^1_0(\Omega)=\overline{{\rm span}_{n\in\mathbb{N}}\{\varphi_n\}}$ and
the space $E^r$ could be expressed by
$$
E^r=D(A^r)=\big\{u=\sum_{n=1}^{+\infty} a_n\varphi_n\in L^2(\Omega):
 \sum_{n=1}^{+\infty}\mu_n^r a_n^2<+\infty\big\}.
$$

\section{Existence of infinitely many solutions of \eqref{eq:1.1}}

In this section, we do the existence of infinitely many solutions of
\eqref{eq:1.1}. We first  recall one type of linking theorem in
\cite{PW} (see also \cite{F}) that provides us with infinitely many
critical points of $I$. We split Hilbert space $E=X\oplus Y$ where
$X$ and $Y$ are both infinite dimensional subspaces.  Assume there
exists  a sequence of finite dimensional subspaces $X_n\subset X$,
$Y_n\subset Y$, $E_n=X_n\oplus Y_n$ such that
$\overline{\cup_{n=1}^{\infty}E_n}=E$. Let $T: E\to E$ be a linear
bounded invertible operator.

We say that the  functional $I$ satisfies the $(PS)^*$ condition
with respect to $E_n$, if any sequence $ \{\mathbf{u}_j\}\subset
{E_{n_j}} $ with $n_j\to\infty$ as $j\to+\infty$, such that
$$
|I(\mathbf{u}_j)|\to c\quad\text{and}\quad 
I|'_{E_{n_j}}(\mathbf{u}_j)\to0
$$
possesses a subsequence converging in $E$.

Let $S_\rho=\{y\in Y, \|y\|_E=\rho\}$, fix $y_1\in Y$ with
$\|y_1\|_E=1$ and  subspaces $Z_1$, $Z_2$ such that
$$
X\oplus\operatorname{span}\{y_1\}=Z_1\oplus Z_2\quad \text{and}\quad y_1\in Z_2.
$$
We next define, for $M,\sigma>0$,
 $$
D=D_{M,\sigma}=\{x_1+x_2\in Z_1\oplus Z_2,\|x_1\|_E\le
M, \|x_2\|_E\le \sigma\}.
$$
The following linking theorem is used to prove the existence result for
system \eqref{eq:1.1}.

\begin{theorem}[\cite{PW}] \label{linking} 
Suppose that $I\in C^1(E,\mathbb{R})$ be an even functional. We assume that:
\begin{itemize}
\item[(L1)]  $I$ satisfies $(PS)^*$ condition with respect to $E_n$, 

\item[(L2)] $T:E_n\to E_n$, for $n$ large, and $\sigma,\rho>0$ satisfy 
$\sigma\|T y_1\|_E>\rho$,

\item[(L3)] There are constants $\alpha\le\beta$ such that
$$
\inf_{S_\rho\cap E_n}I\ge \alpha,\ \sup_{T(\partial D\cap E_n)}I<\alpha
\quad \text{and}\quad \sup_{T(D\cap E_n)}I\le\beta
$$
for all $n$ large.
\end{itemize}
 Then $I$ has a  critical value $c\in[\alpha,\beta]$.
 \end{theorem}

To apply  Theorem \ref{linking} for  solving our problem,
we recall that the functional $I$ is defined in \eqref{4} in
$E(\Omega)$, which is a product Hilbert space defined by
\[
E(\Omega)=E^r(\Omega)\times E^{2-r}(\Omega), \quad  0<r<2,
\]
with the norm
$$
\|\mathbf{u}\|^2_E=\int_\Omega |A^ru|^2+|A^{2-r}v|^2dx
=\|u\|^2_{E^r}+\|v\|^2_{E^{2-r}},
\quad  \mathbf{u}=(u,v)\in E(\Omega),
$$
which is induced by inner product
$$
\langle\mathbf{u},\mathbf{w}\rangle_E
=\langle u,\varphi\rangle_{E^r}+\langle v,\psi\rangle_{E^{2-r}},
$$
where $\mathbf{u}=(u,v)\in E(\Omega)$ and
$\mathbf{w}=(\varphi,\psi)\in E(\Omega)$.

We define $E^+:=\{(u,A ^{r-2}A^ru)|\ u\in E^r \}$ and 
$ E^-:=\{(u,-A ^{r-2}A^ru)|\ u\in E^r \}$, then $E$ has orthogonal decomposition:
$$
E(\Omega)=E^+\oplus E^-=\{\mathbf{u}=\mathbf{u}^++\mathbf{u}^-,
\mathbf{u}^\pm \in E^\pm \}.
$$
 Let
$$
E_n=\operatorname{span}\{\varphi_1,\dots ,\varphi_n\}\times
\operatorname{span}\{\varphi_1,\dots ,\varphi_n\}.
$$
We first prove that $I$ satisfies the $(PS)^*$ condition with
respect to $E_n$.

\begin{lemma}\label{ps*}
The functional $I$ satisfies $(PS)^*$ condition with respect to
$E_n$.
\end{lemma}

\begin{proof}
  Suppose $\{\mathbf{u}_{j}\}\subset E_{n_j}$
be a sequence such that
$$
I(\mathbf{u}_j)\to c , \quad  I|'_{E_{n_j}}(\mathbf{u}_j)\to0.
$$
 We claim that  $(\mathbf{u}_j)$ is
bounded in $E$. Taking $\mathbf{w}=\mathbf{u}_j$, we obtain for a
sequence positive numbers $\epsilon_j\to0$ as $j\to+\infty$
 \begin{align*}
C+\varepsilon_j\|\mathbf{u}_j\|_E
&\geq I(\mathbf{u}_j)-\frac{1}{2}\langle I'(\mathbf{u}_j),\mathbf{u}_j\rangle
\\
&=\int_\Omega \big[(\frac12-\frac{1}{p_t+1})\frac{|v_j|^{p_t+1}}{|y|^t}+
(\frac12-\frac{1}{p_s+1})\frac{|u_j|^{p_s+1}}{|y|^s}\big]dx.
\end{align*}
Then we have
\[
 \int_\Omega \frac{|v_j|^{p_t+1}}{|y|^t}dx
+\int_\Omega \frac{|u_j|^{p_s+1}}{|y|^s}dx\leq C+\varepsilon_j\|\mathbf{u}_j\|_E.
\]
 Noting $\mathbf{u}_j =\mathbf{u}^+_j+\mathbf{u}^-_j$,
$\mathbf{u}^\pm_j=(u^\pm_j ,v^\pm_j)$, we have
\[
\|\mathbf{u}^\pm_j \|^2_E-\varepsilon_j\|\mathbf{u}^\pm_j\|_E
 \le|\int_\Omega(\frac{|v_j|^{p_t-1}}{|y|^t}v_jv^\pm_j
+\frac{|u_j |^{p_s-1}}{|y|^s}u_ju^\pm_j )dx|.
\]
By H\"{o}lder's inequality,
 \begin{align*}
|\int_\Omega\frac{|v_j|^{p_t-1}}{|y|^t}v_jv^\pm_j  dx| &\leq
(\int_\Omega\frac{|v_j
|^{p_t+1}}{|y|^t}dx)^\frac{p_t}{p_t+1}(\int_\Omega \frac{|v^\pm_j
|^{p_t+1}}{|y|^t}dx)^\frac{1}{p_t+1}
\\&\leq(\int_\Omega\frac{|v_j |^{p_t+1}}{|y|^t}dx)^\frac{p_t}{p_t+1}\|v^\pm_j \|_{E^{2-r}}
\\&\leq(\int_\Omega\frac{|v_j |^{p_t+1}}{|y|^t}dx)^\frac{p_t}{p_t+1}\|\mathbf{u}^\pm_j\|_E  .
\end{align*}
Similarly, we have
 \begin{align*}
|\int_\Omega \frac{|u_j|^{p_s-1}}{|y|^s}u_ju^\pm_j  dx| &\leq
(\int_\Omega\frac{|u_j
|^{p_s+1}}{|y|^s}dx)^\frac{p_s}{p_s+1}\|u^\pm_j\|_{E^r}
\\&\leq (\int_\Omega\frac{|u_j |^{p_s+1}}{|y|^s}dx)^\frac{p_s}{p_s+1}\|\mathbf{u}^\pm_j\|_E.
\end{align*}
Then we obtain
 \begin{align*}
\|\mathbf{u}^\pm_j \|_E-\varepsilon_j &\leq (\int_\Omega\frac{|v_j
|^{p_t+1}}{|y|^t}dx)^\frac{p_t}{p_t+1}+(\int_\Omega\frac{|u_j
|^{p_s+1}}{|y|^s}dx)^\frac{p_s}{p_s+1}
\\& \leq(C+\varepsilon_j \|\mathbf{u}_j\|_E)^\frac{p_t}{p_t+1}+(C+\varepsilon_j \|\mathbf{u}_j\|_E)^\frac{p_s}{p_s+1},
\end{align*}
which yields $\|\mathbf{u}_j\|_E\leq C$ uniformly in $j$.

By Theorem \ref{th4.2}, $u_j$ and $v_j $ have subsequences which
converge strongly in $L^{p_s+1}_s$ and $L^{p_t+1}_t$, respectively.
Then we obtain $\mathbf{u}_j=(u_j ,v_j)$ possesses a subsequence
converging in a standard way. Hence $I$ satisfies $(PS)^*$ condition
with respect to $E_n$.
\end{proof}


Now  fix $j$, and we split $E_n$ into $X_n\oplus Y_n$, where
 $X_n=(E_1^-\oplus \dots\oplus  E_n^-)\oplus(E_1^+\oplus \dots
\oplus E_{j-1}^+)$ and $Y_n=E_j^+\oplus \dots \oplus
E_n^+$ with $E^+_i=\operatorname{span}\{(\varphi_i,A^{r-2}A^r\varphi_i)\}$,
$E^-_i=\operatorname{span}\{(\varphi_i,-A^{r-2}A^r\varphi_i) \}$.  Next we define,
for $\mathbf{u}=(u ,v)\in E$
\begin{equation}\label{L}
T_\sigma(\mathbf{u})=(\sigma^{\mu-1}u,\sigma^{\nu-1}v),
\end{equation}
where $\mu,\nu>1$ will be chosen latter. By \eqref{L}, (L2) holds
for $T$ and $y_1=(\varphi_j,A^{r-2}A^r\varphi_j)$.

In what follows, we prove (L3) under our assumptions above.

\begin{lemma}\label{alphaj}
There exist $\alpha_j>0$ and $\rho_j>0$ independent of $n$ such that
for all $n\ge j$
$$
\inf_{S_{\rho_j}\cap Y_n}I\ge \alpha_j,
$$
where $Y=E_j^+\oplus \dots  \oplus E_n^+\oplus
\dots $ and $S_{\rho_j}=\{y\in Y,\|y\|_E=\rho_j\}$.
Moreover, $\alpha_j\to+\infty$ as $j\to+\infty$.
\end{lemma}

\begin{proof} 
For $\mathbf{u}=(u,v)\in Y $, we have
$$
\|u\|^2_{E^r}\ge \mu_j^{\min\{r,2-r\}}\|u\|^2_{L^2}\quad
\text{and}\quad \|v\|^2_{E^{2-r}}\ge \mu_j^{\min\{r,2-r\}}\|v\|^2_{L^2}.
$$
By Theorem \ref{th4.2} and H\"{o}lder inequality, we have that
$$
\|u\|_{L^{p_s+1}_s}^{p_s+1}\le
\|u\|_{L^2}^{2\kappa}\|u\|_{L^{\frac{
p_s+1-2\kappa}{1-\kappa}}_{\frac{s}{1-\kappa}}}^{p_s+1-2\kappa}\le\frac{C}
{\mu_j^{\min\{r,2-r\}\kappa}}\|u\|_{E^r}^{p_s+1}\le\frac{C}
{\mu_j^{\min\{r,2-r\}\kappa}}\|\mathbf{u}\|_{E}^{p_s+1}
$$ 
and
 $$
\|v\|_{L^{p_t+1}_t}\le\frac{C}
{\mu_j^{\min\{r,2-r\}\bar\kappa}}\|v\|_{E^{2-r}}^{p_t+1}\le\frac{C}
{\mu_j^{\min\{r,2-r\}\bar\kappa}}\|\mathbf{u}\|_{E}^{p_t+1}
$$ 
for some constants $\kappa,\bar\kappa\in(0,1)$ such that
$$
E^r\hookrightarrow L^{\frac{
p_s+1-2\kappa}{1-\kappa}}_{\frac{s}{1-\kappa}}(\Omega)\quad\text{and}\quad
\ E^{2-r}\hookrightarrow L^{\frac{ p_t+1-2\bar
\kappa}{1-\bar\kappa}}_{\frac{t}{1-\bar\kappa}}(\Omega)
$$
are continuous, and $C>0$ independent of $n$. Then we have
that for $\mathbf{u}=(u,v)\in Y $,
 \begin{align*}
I(\mathbf{u})
&=\int_\Omega(|A^ru|^2-\frac{1}{p_t+1}\frac{|v|^{p_t+1}}{|y|^t}
 -\frac{1}{p_s+1}\frac{|u|^{p_s+1}}{|y|^s})dx \\
&\ge\frac12\|\mathbf{u}\|_E^2-\frac{C}
{\mu_j^{\min\{r,2-r\}\min\{\kappa,\bar\kappa\}}}(\|\mathbf{u}\|_{E}^{p_s+1}
 +\|\mathbf{u}\|_{E}^{p_t+1}).
\end{align*}
By choosing
$2\rho_j^{\max\{p_s+1,p_t+1\}}=\mu_j^{\min\{r,2-r\}\min\{\kappa,\bar\kappa\}}$,
we have for $\mathbf{u}\in S_{\rho_j}\cap Y_n$
$$
I(\mathbf{u})\ge\frac12 \rho_j^2-C=:\alpha_j,
$$
and we finished the proof.
\end{proof}

\begin{lemma}\label{beta j}
There exist $\beta_j\ge \alpha_j$, $M_j>0$ and $\sigma_j>\rho_j$
independent of $n$ such that for all $n\ge j$
$$
\sup_{T_{\sigma_j}(\partial D\cap E_n)}I<\alpha_j\quad\text{and}\quad
\sup_{T_{\sigma_j}(D\cap E_n)}I\le\beta_j,
$$
where 
$$
D=\{\mathbf{u}\in E^-\oplus E^+_1\oplus\dots
\oplus\ E^+_j, \|\mathbf{u}^-\|_E\le M_j, \|\mathbf{u}^+\|_E\le
\sigma_j\}.
$$
\end{lemma}

\begin{proof} 
 Let $\mathbf{z}=T_{\sigma_j}(\mathbf{u})$ with
$\mathbf{u}\in D$. Then we can write
$$
\mathbf{z}=(\sigma_j^{\mu-1}u^+,\sigma_j^{\nu-1}A^{r-2}A^{r}u^+)
+(\sigma_j^{\mu-1}u^-,-\sigma_j^{\nu-1}A^{r-2}A^{r}u^-),
$$
where $\mu,\nu>1$ will be chosen latter, $u^+$ and $u^-$ can be
written as
$$
u^+=\sum_{i=1}^{j}\theta_i \varphi_i \quad\text{and}\quad
u^-=\sum_{i=1}^{j}\gamma_i \varphi_i+\tilde u^-
$$
 where $\tilde u^-$
is orthogonal to $\varphi_i$, $i=1,\dots ,j$ in $L^2(\Omega)$. Using
Holder's inequality and  the equivalence of all the norms in finite
dimensional space, we get
\begin{equation}\label{4.1}
\sum_{i=1}^{j}\mu_i^{2r-2}(\theta_i^2+\theta_i\gamma_i)
=\langle u^++u^-,A^{r-2}A^ru^+\rangle \leq
C_j\|u^++u^-\|_{L^{p_s+1}_s}\|u^+\|_{L^2}
\end{equation}
and
\begin{equation}\label{4.2}
\sum_{i=1}^{j}\mu_i^{2r-2}(\theta_i^2-\theta_i\gamma_i)
=\langle v^++v^-,A^{r-2}A^rv^+\rangle \leq
C_j\|v^++v^-\|_{L^{p_t+1}_t}\|u^+\|_{L^2}.
\end{equation}
If $\sum_{i=1}^{j}\theta_i\gamma_i\ge0$, then \eqref{4.1} implies 
 $$
\|u^+\|_{L^2}\leq C_j\|u^++u^-\|_{L^{p_s+1}_s}=C_j\|u\|_{L^{p_s+1}_s},
$$
otherwise, \eqref{4.2} implies 
$$
\|u^+\|_{L^2}\leq
C_j\|v^++v^-\|_{L^{p_t+1}_t}=C_j\|v\|_{L^{p_t+1}_t}.
$$
Hence,
$$
I(\mathbf{u})\le \frac12\sigma_j^{\mu+\nu-2}(\|\mathbf{u}^+\|^2_E
-\|\mathbf{u}^-\|^2_E)-C_j\sigma_j^{(p_s+1)(\mu-1)}\|u^+\|^{p_s+1}_{L^2}
$$
or 
$$
I(\mathbf{u})\le\frac12
\sigma_j^{\mu+\nu-2}(\|\mathbf{u}^+\|^2_E-\|\mathbf{u}^-\|^2_E)
-C_j\sigma_j^{(p_t+1)(\nu-1)}\|u^+\|^{p_t+1}_{L^2}.
$$
Thus we may choose $\|\mathbf{u}^+\|_E=\sigma_j$ large enough in
order to obtain $\sigma_k>\rho_k$ and it is possible to choose
$\mu,\nu>1$ such that $(p_t+1)(\mu-1)>\mu+\nu-2$ and
$(p_s+1)(\nu-1)>\mu+\nu-2$ if 
$$
\frac1{p_t+1}+\frac1{p_s+1}<1,
$$
$p_t,p_s>1$ makes sure that the  estimate above holds. Then,
$I(\mathbf{u})\leq 0$.

Taking $\|\mathbf{u}^+\|_E\le\sigma_j$ and $\|\mathbf{u}^-\|_E=M_j$,
we obtain
 $$
I(z)\le\sigma_j^{\mu+\nu-2}(\sigma_j^2-M_j^2)\leq0
$$ 
if $M_j\ge \sigma_j$.

Then we choose $\beta_j$ large so that the second inequality holds.
\end{proof}

 \subsection*{Proof of existence of infinitely many solutions in Theorem \ref{th1.1}}
 Combining Lemma \ref{ps*}, Lemma \ref{alphaj} with Lemma \ref{beta j},
   $I$ satisfies the conditions (L1)--(L3). By Theorem
   \ref{linking},
 $I$ has a sequence of critical values in $[\alpha_j,\beta_j]$ and
 $\alpha_j\to+\infty$ as $j\to+\infty$,
 then there exist a sequence critical points of $I$, which are infinite 
many solutions of  \eqref{eq:1.1}.
 We finish the proof.

\section{Nonexistence result}

In this section, we show the nonexistence of solution in  Theorem
\ref{th1.1}. To obtain this nonexistence result, we
introduce some lemmas.

Assume that the Euler-Lagrange equations are
\begin{equation} \label{eq:2.1}
div(\frac{\partial L}{\partial p^k_i})-\frac{\partial L}{\partial
u_k}=0,\ \ k=1,\dots,s .
\end{equation}
 $ i=1,\dots,N$;
where $\mathbf{u}=(u_k)$, $\mathbf{p}=(p^k_i)$,
$p^k_i=\frac{\partial u_k}{\partial x^i}$, and $\Omega$ is a bounded and
smooth domain in $\mathbb{R}^N$. We have the following result.


\begin{lemma}[\cite{V}]\label{lemma2.1}
Let $L\in C^1(\Omega\times \mathbb{R}^s\times \mathbb{R}^{N\times s})$ and 
$\mathbf{u}=(u_1,\dots,u_s):\Omega\to \mathbb{R}^s$ be a solution of 
$\eqref{eq:2.1}$ with $u_k\in C^2(\Omega)$. Let $a_{kl},h^i\in C^1(\Omega)$. Then
\begin{equation} \label{eq:2.2}
\begin{aligned}
& \operatorname{div}(h^iL-h^j\frac{\partial u_k}{\partial x^j}\frac{\partial
L}{\partial p^k_i}-a_{kl}u_l\frac{\partial L}{\partial p^k_i})
\\
& =\frac{\partial h^i}{\partial x^i}L+h^i\frac{\partial L}{\partial x^i}-
(\frac{\partial u_k}{\partial x^j}\frac{\partial h^j}{\partial
x^i}+u_l\frac{\partial a_{kl}}{\partial x^i})\frac{\partial
L}{\partial p^k_i} -a_{kl}(\frac{\partial u_l}{\partial
x^i}\frac{\partial L}{\partial p^k_i}+u_l\frac{\partial L}{\partial
u_k}), \quad\text{in } \Omega.
\end{aligned}
\end{equation}
Furthermore,
\begin{equation} \label{eq:2.3}
\begin{aligned}
& \oint_{\partial \Omega}((h^iL-h^j\frac{\partial u_k}{\partial
x^j}\frac{\partial L}{\partial p^k_i}-a_{kl}u_l\frac{\partial
L}{\partial p^k_i}),n)ds
\\
& =\int_\Omega(\frac{\partial h^i}{\partial x^i}L+h^i\frac{\partial L}{\partial x^i}-
(\frac{\partial u_k}{\partial x^j}\frac{\partial h^j}{\partial
x^i}+u_l\frac{\partial a_{kl}}{\partial x^i})\frac{\partial
L}{\partial p^k_i} -a_{kl}(\frac{\partial u_l}{\partial
x^i}\frac{\partial L}{\partial p^k_i}+u_l\frac{\partial L}{\partial
u_k}))dx,
\end{aligned}
\end{equation}
where $n$ is the outward normal on $\partial \Omega$.
\end{lemma}

By choosing suitable functions in the above lemma, we obtain the following
result.

\begin{lemma}\label{th2.1}
Let $(u,v)\in (C^2(\Omega)\cap C^1(\overline{\Omega}))^2$ be a
solution of problem  \eqref{eq:1.1}. Then $u$ and $v$ satisfy the
identity
\begin{eqnarray}\label{eq:2.5}
\begin{aligned}
 \oint_{\partial \Omega}(\nabla u\cdot \nabla v)(x, n)ds
&=\int_\Omega\{(2+a_{11}+a_{22}-N)(\nabla u\cdot \nabla
v-\frac{\lambda uv}{|y|^2})\\
&\quad +(\frac{N-t}{p_t+1}-a_{22})\frac{|v|^{p_t+1}}{|y|^t}
+(\frac{N-s}{p_s+1}-a_{11})\frac{|u|^{p_s+1}}{|y|^s}\} dx,
\end{aligned}
\end{eqnarray}
where $a_{11}$ and $a_{22}$ are constants to be chosen latter.
\end{lemma}

\begin{proof} For our system \eqref{eq:1.1}, we define
\begin{equation} \label{eq:2.4}
L=\Sigma_{i=1}^N  p^1_ip^2_i-\lambda\frac{uv}{|y|^2}-\frac
1{p_t+1}\frac{|v|^{p_t+1}}{|y|^t}-\frac
1{p_s+1}\frac{|u|^{p_s+1}}{|y|^s},
\end{equation}
where $p^1_i=\frac{\partial u}{\partial x^i},\ p^2_i=\frac{\partial
v}{\partial x^i} $, $x^i=y^i$ if $i\leq k$ and $(u,v)$ is a
classical solution of system \eqref{eq:1.1}.

We give explicitly the values of parameters for using  Lemma
\ref{lemma2.1}: $k=1,2$, $a_{11}(x)=a_{11},a_{22}(x)=a_{22}$, $
a_{12}(x)=a_{21}(x)=0$ and $h^i(x)=x^i$ where $ i=1,\dots, N$. For
 the purpose of deleting the  singularity of $L$ at
the domain $U=\{x=(y,z) \in \Omega:y=0\}$, assume that
$N_\delta(U)=\{x\in \Omega:dist(x,U)\leq \delta\}$ and
$\Omega_\delta=\Omega\setminus N_\delta(U), $ where $\delta>0 $. And
we have
 $\partial \Omega_\delta=(\partial \Omega\setminus  \partial
N_\delta(U))\cup( \partial N_\delta(U)\setminus\partial \Omega)$.


Since $u(x)=v(x)=0$, $x\in \partial\Omega$,  we have
$$
x^j\frac{\partial u}{\partial x^j}\frac{\partial v}{\partial x^i}n_i
=\frac{\partial u}{\partial x^i}\frac{\partial v}{\partial
x^i}x^jn_j, 
$$ 
which follows from 
$\frac{\partial u}{\partial x^i}=\frac{\partial u}{\partial n}n_i$,
$\frac{\partial v}{\partial x^i}=\frac{\partial v}{\partial n}n_i$.
Then the left-hand side of \eqref{eq:2.3} is
\begin{align*}
&\oint_{ \partial N_\delta(U)\setminus\partial \Omega}(\nabla
u\cdot\nabla v-\lambda\frac{uv}{|y|^2}-\frac
1{p_t+1}\frac{|v|^{p_t+1}}{|y|^t}-\frac
1{p_s+1}\frac{|u|^{p_s+1}}{|y|^s})(x,n)ds\\
&-\oint_{ \partial
N_\delta(U)\setminus\partial \Omega}((\Sigma_{j=1}^N
x^j\frac{\partial u}{\partial x^j}\frac{\partial v}{\partial
x^i}+\Sigma_{j=1}^Nx^j\frac{\partial v}{\partial x^j} \frac{\partial
u}{\partial x^i}+a_{11}u\frac{\partial v}{\partial
x^i}+a_{22}v\frac{\partial u}{\partial x^i}),n)ds\\
&-\oint_{\partial \Omega\setminus  \partial N_\delta(U)}(\nabla
u\cdot\nabla v)(x,n)ds.
\end{align*}

We claim that the first two terms in the quantity above go to zero
as $\delta\to0$. In fact,  $|\nabla u|$, $|\nabla v|$, $u$
and $v$ are bounded and $\lim_{\delta\to0}|\partial
N_\delta(U)\setminus\partial \Omega|=0$, we have the first two terms
go to zero. For the third term, since $\lim_{\delta\to0}\partial
\Omega\setminus  \partial N_\delta(U)=\partial
\Omega\setminus\{(0,z)\in\partial \Omega\}$ and $|\partial
\Omega\setminus\{(0,z)\in\partial \Omega\}|=|\partial \Omega|$, we
obtain that this term tends to 
$$
-\oint_{\partial \Omega}(\nabla u\cdot\nabla v)(x,n)ds.
$$ 
Hence,  the left hand of \eqref{eq:2.3} tends to
$$
-\oint_{\partial \Omega}(\nabla u\cdot\nabla v)(x,n)ds.
$$

In the following, we do estimate the right hand side of
\eqref{eq:2.3}. After calculating,  the right hand side of
\eqref{eq:2.3} with integrate domain being $\Omega_\delta$,
\begin{align*}
&\int_{\Omega_\delta}
\big\{(N-2-a_{11}-a_{22})(\nabla u\cdot\nabla v
-\frac{\lambda uv}{|y|^2})
-(\frac{N-t}{p_t+1}-a_{22})\frac{|v|^{p_t+1}}{|y|^t}\\
&-(\frac{N-s}{p_s+1}-a_{11})\frac{|u|^{p_s+1}}{|y|^s}\}dx.
\end{align*}
Since $\lim_{\delta\to0^+}\Omega_\delta=\Omega\setminus U$ and
$|\Omega\setminus U|=|\Omega|$, the right-hand side of
\eqref{eq:2.3} tends to
\begin{align*}
&\int_{\Omega}\{(N-2-a_{11}-a_{22})(\nabla u\cdot\nabla
v-\frac{\lambda
uv}{|y|^2})-(\frac{N-t}{p_t+1}-a_{22})\frac{|v|^{p_t+1}}{|y|^t}\\
&-(\frac{N-s}{p_s+1}-a_{11})\frac{|u|^{p_s+1}}{|y|^s}\}dx.
\end{align*}
 Thus, using the Lemma \ref{lemma2.1}, this yields \eqref{eq:2.5}.
\end{proof}

Now we use Lemma \ref{th2.1} to obtain the following nonexistence
result.

\begin{theorem}\label{th2}
Suppose that $\Omega$ is star-sharped with respect to the origin.
 Let $ 0\leq\lambda<\frac{(k-2)^2}{4}$ if $k>2$,
$\lambda=0$ if $k=2$,  $0\leq t,s <2$ and $p_t,p_s>1$ satisfying
\begin{equation}\label{3}
\frac{1}{p_t+1}(1-\frac{t}{N})+\frac{1}{p_s+1}(1-\frac{s}{N})\leq\frac{N-2}{N}.
\end{equation}
Then system \eqref{eq:1.1} does not have any classical positive
solution.
\end{theorem}

\begin{proof} 
 Suppose $(u,v)$ is classical positive
solution of system \eqref{eq:1.1}, then $u(x)=v(x)=0$, $x\in
\partial\Omega$. Since $\Omega$ is star-shaped with respect to the
origin, then $(x^0,n)\geq 0$ for all $x^0\in\partial\Omega$ and
$(x^0,n)>0$ on some subset of $\partial\Omega$ of positive measure
(see \cite{PJ}). And applying Hopf's Lemma (see \cite{E}), we have
$$
\frac{\partial u}{\partial n}(x^0)=(\nabla u,n)|_{x=x^0}<0,\quad
\frac{\partial v}{\partial n}(x^0)=(\nabla v,n)|_{x=x^0}<0,
$$
and $\frac{\partial u}{\partial x^i}=\frac{\partial u}{\partial
n}n_i$, $\frac{\partial v}{\partial x^i}=\frac{\partial v}{\partial
n}n_i$ when $x\in \partial\Omega$, which implies
\[
(\nabla u\cdot \nabla v)|_{x=x^0}=\frac{\partial u}{\partial
n}\frac{\partial v}{\partial n}|_{x=x^0}>0.
\]
Then the left-hand side of \eqref{eq:2.5} in  Lemma \ref{th2.1} is
$$
\oint_{\partial \Omega}(\nabla u\cdot \nabla v)(x, n)ds>0,
$$
and now by choosing $a_{11}=N-2-\frac{N-t}{p_t+1}$, 
$a_{22}=\frac{N-t}{p_t+1}$ in \eqref{eq:2.5}, it yields
\[
\int_\Omega[\frac{N-s}{p_s+1}-(N-2-\frac{N-t}{p_t+1})]
\frac{|u|^{p_s+1}}{|y|^s}dx>0.
\]
Therefore, we obtain $p_s,p_t$ satisfy the  formulation
\[
\frac{N-s}{p_s+1}-(N-2-\frac{N-t}{p_t+1})>0,
\]
which contradicts \eqref{3}. This complete the proof of Theorem
\ref{th2}.
\end{proof}

\begin{thebibliography}{00}


\bibitem {AE} A. Ambrosetti, E. Colorado: 
\emph{Bound and ground states of
coupled nonlinear Schr\"{o}dinger equations,} C. R. Acad. Sci. Pairs
S\'{e}r. I Math.,\textbf{ 342 } (2006), 453-458.

\bibitem {BG} M. Badiale, G. Tarantello:
 \emph{A Sobolev-Hardy inequality with
applications to a nonlinnonlinear elliptic equation arising in
astrophysics,} Arch. Ration. Mech. Anal., \textbf{163} (2002),
259-293.

\bibitem {MK} M. Bhakta, K. Sandeep: 
\emph{Hardy-Sobolev-Maz'ya type equations in bounded domains,} 
J. Differential Equations,\textbf{247}
(2009), 119-139.

\bibitem{CP} D. Cao, S. Peng: 
\emph{A global compactness result for
singular elliptic problems involving critical Sobolev exponent,}
Proc. Amer. Math. Soc., \textbf{ 131} (6) (2003), 1857-1866.

\bibitem {DI} D. Castorina, I. Fabbri, G. Mancini, K. Sandeep: 
\emph{Hardy-Sobolev extremals, hyperbolic symmetry and scalar curvature
equations,} J. Differential Equations, \textbf{246} (2009),
1187-1206.

\bibitem{F} D. G. deFigueiredo, I. Peral,  J. D. Rossi:
\emph{The critical hyperbola for a Hamilton
elliptic system with weights,}  Annali di Matematica Pura ed
Applicata {\bf 187} (2008), 531-545.

\bibitem {E} L. C. Evans:
 \emph{Partial Differential Equations,}
Amer. Math. Soc., 1998.

\bibitem{CH} I. Fabbri, G. Mancini, K. Sandeep: 
\emph{Classification of solutions
of a critical Hardy-Sobolev operator,} J. Differential Equations,
\textbf{224} (2) (2006), 258-276.

\bibitem {PW} P. Felmer, Z-Q. Wang:
 \emph{Multiplicity for symmetric indefinite funtionals: application
to hamiltonian and elliptic systems, }
Topol. Methods Nonlinear Anal., {\bf 12}(2)(1998), 207-226.

\bibitem {JR} J. Hulshof, R. Vandervorst: 
\emph{Differential systems with strongly indefinite variational structure,} 
J. Funct. Anal., \textbf{114} (1993), 32-58.

\bibitem {L} S. Li, M. Willem:
 \emph{Applications of local linking to critical point
theory,} J. Math. Anal. Appl., \textbf{189} (1995), 6-32.

\bibitem{LE} J. L. Lion, E. Magens:
 \emph{Non-homogeneous boundary value
problems and applications,} Vol I. New York Heidelberg, Berlin:
Springer-Verlag, 1972.

\bibitem {LY} F. Liu, J. Yang:
 \emph{Nontrivial solutions of Hardy-H\'{e}non
type elliptic systems,}  Acta Mathematics Scientia, \textbf{27}
(2007), 673-688.

\bibitem {VM} V. G. Maz'ya:
\emph{Sobolev Spaces,} Springer Ser. Soviet Math, Springer-Verlag,
Berlin, 1985.

\bibitem {P} A. Persson:
 \emph{Compact linear mappings between interpolation spaces,} 
Arch. Math., \textbf{5} (1964), 215-219.

\bibitem{PJ} P. Pucci, J. Serrin: 
\emph{A general variational identity,} Indiana Univ. Math. J., \textbf{35}
(1986), 681-703.

\bibitem {C} C. Tarsi: 
\emph{Perturbation of symmetry and multiplicity
 of solutins for strongly indefinite elliptic systems,} Advanced Nonlinear
 Studies, \textbf{7} (2007), 1-30.

\bibitem{V}  R. Vandervorst: 
\emph{Variational identities and applications to differential
systems,} Arch. Rational Mech. Anal., \textbf{ 116} (1991), 375-398.

\bibitem {MM} M. Willem: \emph{Minimax Theorems,}  birkh\"{a}user, Basel, 1996.

\end{thebibliography}

 \end{document}


